blob: c0719d4334e1ebce0a904023480fb1f130fe655c [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010030#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020031#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070032#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080033#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020034#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020035#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010036#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020038#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020039#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080040#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020041#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010042#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020043#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020044#include "system_wrappers/include/sleep.h"
45#include "test/encoder_settings.h"
46#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020047#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010048#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020049#include "test/gmock.h"
50#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020051#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020052#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070053
54namespace webrtc {
55
sprang57c2fff2017-01-16 06:24:02 -080056using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020057using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020058using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020060using ::testing::Ge;
61using ::testing::Gt;
62using ::testing::Le;
63using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010064using ::testing::Matcher;
65using ::testing::NiceMock;
66using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020067using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080068
perkj803d97f2016-11-01 11:45:46 -070069namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020070const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010071const int kQpLow = 1;
72const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020073const int kMinFramerateFps = 2;
74const int kMinBalancedFramerateFps = 7;
75const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080076const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010077const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020078const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010079const uint32_t kSimulcastTargetBitrateBps = 3150000;
80const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080081const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070082const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020083const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020084const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020085const VideoEncoder::ResolutionBitrateLimits
86 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
87const VideoEncoder::ResolutionBitrateLimits
88 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080089
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020090uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
91 0x00, 0x00, 0x03, 0x03, 0xF4,
92 0x05, 0x03, 0xC7, 0xE0, 0x1B,
93 0x41, 0x10, 0x8D, 0x00};
94
perkj803d97f2016-11-01 11:45:46 -070095class TestBuffer : public webrtc::I420Buffer {
96 public:
97 TestBuffer(rtc::Event* event, int width, int height)
98 : I420Buffer(width, height), event_(event) {}
99
100 private:
101 friend class rtc::RefCountedObject<TestBuffer>;
102 ~TestBuffer() override {
103 if (event_)
104 event_->Set();
105 }
106 rtc::Event* const event_;
107};
108
Noah Richards51db4212019-06-12 06:59:12 -0700109// A fake native buffer that can't be converted to I420.
110class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
111 public:
112 FakeNativeBuffer(rtc::Event* event, int width, int height)
113 : event_(event), width_(width), height_(height) {}
114 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
115 int width() const override { return width_; }
116 int height() const override { return height_; }
117 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
118 return nullptr;
119 }
120
121 private:
122 friend class rtc::RefCountedObject<FakeNativeBuffer>;
123 ~FakeNativeBuffer() override {
124 if (event_)
125 event_->Set();
126 }
127 rtc::Event* const event_;
128 const int width_;
129 const int height_;
130};
131
Niels Möller7dc26b72017-12-06 10:27:48 +0100132class CpuOveruseDetectorProxy : public OveruseFrameDetector {
133 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200134 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
135 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200136 last_target_framerate_fps_(-1),
137 framerate_updated_event_(true /* manual_reset */,
138 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100139 virtual ~CpuOveruseDetectorProxy() {}
140
141 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200142 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100143 last_target_framerate_fps_ = framerate_fps;
144 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200145 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100146 }
147
148 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200149 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100150 return last_target_framerate_fps_;
151 }
152
Niels Möller4db138e2018-04-19 09:04:13 +0200153 CpuOveruseOptions GetOptions() { return options_; }
154
Henrik Boström381d1092020-05-12 18:49:07 +0200155 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
156
Niels Möller7dc26b72017-12-06 10:27:48 +0100157 private:
Markus Handella3765182020-07-08 13:13:32 +0200158 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100159 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200160 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100161};
162
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200163class FakeVideoSourceRestrictionsListener
164 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200165 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200166 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200167 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200168 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200169 RTC_DCHECK(was_restrictions_updated_);
170 }
171
172 rtc::Event* restrictions_updated_event() {
173 return &restrictions_updated_event_;
174 }
175
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200176 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200177 void OnVideoSourceRestrictionsUpdated(
178 VideoSourceRestrictions restrictions,
179 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200180 rtc::scoped_refptr<Resource> reason,
181 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200182 was_restrictions_updated_ = true;
183 restrictions_updated_event_.Set();
184 }
185
186 private:
187 bool was_restrictions_updated_;
188 rtc::Event restrictions_updated_event_;
189};
190
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200191auto WantsFps(Matcher<int> fps_matcher) {
192 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
193 fps_matcher);
194}
195
196auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
197 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
198 AllOf(max_pixel_matcher, Gt(0)));
199}
200
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200201auto ResolutionMax() {
202 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200203 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200204 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
205 Eq(absl::nullopt)));
206}
207
208auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200209 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200210}
211
212auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200213 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200214}
215
216auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200217 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200218}
219
220auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200221 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200222}
223
224auto FpsMaxResolutionMax() {
225 return AllOf(FpsMax(), ResolutionMax());
226}
227
228auto UnlimitedSinkWants() {
229 return AllOf(FpsUnlimited(), ResolutionMax());
230}
231
232auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
233 Matcher<int> fps_range_matcher;
234
235 if (last_frame_pixels <= 320 * 240) {
236 fps_range_matcher = AllOf(Ge(7), Le(10));
237 } else if (last_frame_pixels <= 480 * 270) {
238 fps_range_matcher = AllOf(Ge(10), Le(15));
239 } else if (last_frame_pixels <= 640 * 480) {
240 fps_range_matcher = Ge(15);
241 } else {
242 fps_range_matcher = Eq(kDefaultFramerate);
243 }
244 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
245 fps_range_matcher);
246}
247
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
249 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
250 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
251}
252
253auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
254 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
255}
256
257auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
258 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
259}
260
261auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
262 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
263 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
264}
265
266auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
267 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
268 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
269}
270
271auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
272 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
273 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
274}
275
276auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
277 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
278 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
279}
280
mflodmancc3d4422017-08-03 08:27:51 -0700281class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700282 public:
Niels Möller213618e2018-07-24 09:29:58 +0200283 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200284 const VideoStreamEncoderSettings& settings,
285 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100286 : VideoStreamEncoder(Clock::GetRealTimeClock(),
287 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200288 stats_proxy,
289 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200290 std::unique_ptr<OveruseFrameDetector>(
291 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100292 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100293 task_queue_factory),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200294 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200295 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200296 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200297 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200298 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200299 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200300 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100301 }
perkj803d97f2016-11-01 11:45:46 -0700302
Henrik Boström381d1092020-05-12 18:49:07 +0200303 void SetSourceAndWaitForRestrictionsUpdated(
304 rtc::VideoSourceInterface<VideoFrame>* source,
305 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200306 FakeVideoSourceRestrictionsListener listener;
307 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200308 SetSource(source, degradation_preference);
309 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200310 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200311 }
312
313 void SetSourceAndWaitForFramerateUpdated(
314 rtc::VideoSourceInterface<VideoFrame>* source,
315 const DegradationPreference& degradation_preference) {
316 overuse_detector_proxy_->framerate_updated_event()->Reset();
317 SetSource(source, degradation_preference);
318 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
319 }
320
321 void OnBitrateUpdatedAndWaitForManagedResources(
322 DataRate target_bitrate,
323 DataRate stable_target_bitrate,
324 DataRate link_allocation,
325 uint8_t fraction_lost,
326 int64_t round_trip_time_ms,
327 double cwnd_reduce_ratio) {
328 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
329 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
330 // Bitrate is updated on the encoder queue.
331 WaitUntilTaskQueueIsIdle();
332 // Give the managed resources time to react to the new bitrate.
333 // TODO(hbos): Can we await an appropriate event instead?
334 WaitUntilAdaptationTaskQueueIsIdle();
335 }
336
337 void WaitUntilAdaptationTaskQueueIsIdle() {
338 rtc::Event event;
339 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
340 ASSERT_TRUE(event.Wait(5000));
341 }
342
kthelgason2fc52542017-03-03 00:24:41 -0800343 // This is used as a synchronisation mechanism, to make sure that the
344 // encoder queue is not blocked before we start sending it frames.
345 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100346 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200347 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800348 ASSERT_TRUE(event.Wait(5000));
349 }
350
Henrik Boström91aa7322020-04-28 12:24:33 +0200351 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200352 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200353 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200354 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200355 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200356 event.Set();
357 });
358 ASSERT_TRUE(event.Wait(5000));
359 }
360 void TriggerCpuUnderuse() {
361 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200362 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200363 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200364 event.Set();
365 });
366 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200367 }
kthelgason876222f2016-11-29 01:44:11 -0800368
Henrik Boström91aa7322020-04-28 12:24:33 +0200369 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200370 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200371 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200372 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200374 event.Set();
375 });
376 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200377 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200378 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200379 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200380 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200381 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 event.Set();
383 });
384 ASSERT_TRUE(event.Wait(5000));
385 }
386
Niels Möller7dc26b72017-12-06 10:27:48 +0100387 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200388 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
389 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200390 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700391};
392
asapersson5f7226f2016-11-25 04:37:00 -0800393class VideoStreamFactory
394 : public VideoEncoderConfig::VideoStreamFactoryInterface {
395 public:
sprangfda496a2017-06-15 04:21:07 -0700396 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
397 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800398 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700399 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800400 }
401
402 private:
403 std::vector<VideoStream> CreateEncoderStreams(
404 int width,
405 int height,
406 const VideoEncoderConfig& encoder_config) override {
407 std::vector<VideoStream> streams =
408 test::CreateVideoStreams(width, height, encoder_config);
409 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100410 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700411 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800412 }
413 return streams;
414 }
sprangfda496a2017-06-15 04:21:07 -0700415
asapersson5f7226f2016-11-25 04:37:00 -0800416 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700417 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800418};
419
Noah Richards51db4212019-06-12 06:59:12 -0700420// Simulates simulcast behavior and makes highest stream resolutions divisible
421// by 4.
422class CroppingVideoStreamFactory
423 : public VideoEncoderConfig::VideoStreamFactoryInterface {
424 public:
425 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
426 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
427 EXPECT_GT(num_temporal_layers, 0u);
428 EXPECT_GT(framerate, 0);
429 }
430
431 private:
432 std::vector<VideoStream> CreateEncoderStreams(
433 int width,
434 int height,
435 const VideoEncoderConfig& encoder_config) override {
436 std::vector<VideoStream> streams = test::CreateVideoStreams(
437 width - width % 4, height - height % 4, encoder_config);
438 for (VideoStream& stream : streams) {
439 stream.num_temporal_layers = num_temporal_layers_;
440 stream.max_framerate = framerate_;
441 }
442 return streams;
443 }
444
445 const size_t num_temporal_layers_;
446 const int framerate_;
447};
448
sprangb1ca0732017-02-01 08:38:12 -0800449class AdaptingFrameForwarder : public test::FrameForwarder {
450 public:
451 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
asapersson09f05612017-05-15 23:40:18 -0700464 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200465 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700466 return last_wants_;
467 }
468
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_sent_width() const { return last_width_; }
470 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800471
sprangb1ca0732017-02-01 08:38:12 -0800472 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
473 int cropped_width = 0;
474 int cropped_height = 0;
475 int out_width = 0;
476 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700477 if (adaption_enabled()) {
478 if (adapter_.AdaptFrameResolution(
479 video_frame.width(), video_frame.height(),
480 video_frame.timestamp_us() * 1000, &cropped_width,
481 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100482 VideoFrame adapted_frame =
483 VideoFrame::Builder()
484 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
485 nullptr, out_width, out_height))
486 .set_timestamp_rtp(99)
487 .set_timestamp_ms(99)
488 .set_rotation(kVideoRotation_0)
489 .build();
sprangc5d62e22017-04-02 23:53:04 -0700490 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100491 if (video_frame.has_update_rect()) {
492 adapted_frame.set_update_rect(
493 video_frame.update_rect().ScaleWithFrame(
494 video_frame.width(), video_frame.height(), 0, 0,
495 video_frame.width(), video_frame.height(), out_width,
496 out_height));
497 }
sprangc5d62e22017-04-02 23:53:04 -0700498 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800499 last_width_.emplace(adapted_frame.width());
500 last_height_.emplace(adapted_frame.height());
501 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200502 last_width_ = absl::nullopt;
503 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700504 }
sprangb1ca0732017-02-01 08:38:12 -0800505 } else {
506 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800507 last_width_.emplace(video_frame.width());
508 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800509 }
510 }
511
512 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
513 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200514 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200515 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100516 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200517 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800518 }
sprangb1ca0732017-02-01 08:38:12 -0800519 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200520 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
521 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200522 absl::optional<int> last_width_;
523 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800524};
sprangc5d62e22017-04-02 23:53:04 -0700525
Niels Möller213618e2018-07-24 09:29:58 +0200526// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700527class MockableSendStatisticsProxy : public SendStatisticsProxy {
528 public:
529 MockableSendStatisticsProxy(Clock* clock,
530 const VideoSendStream::Config& config,
531 VideoEncoderConfig::ContentType content_type)
532 : SendStatisticsProxy(clock, config, content_type) {}
533
534 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200535 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700536 if (mock_stats_)
537 return *mock_stats_;
538 return SendStatisticsProxy::GetStats();
539 }
540
Niels Möller213618e2018-07-24 09:29:58 +0200541 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200542 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200543 if (mock_stats_)
544 return mock_stats_->input_frame_rate;
545 return SendStatisticsProxy::GetInputFrameRate();
546 }
sprangc5d62e22017-04-02 23:53:04 -0700547 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200548 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700549 mock_stats_.emplace(stats);
550 }
551
552 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200553 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700554 mock_stats_.reset();
555 }
556
557 private:
Markus Handella3765182020-07-08 13:13:32 +0200558 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200559 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700560};
561
sprang4847ae62017-06-27 07:06:52 -0700562class MockBitrateObserver : public VideoBitrateAllocationObserver {
563 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200564 MOCK_METHOD(void,
565 OnBitrateAllocationUpdated,
566 (const VideoBitrateAllocation&),
567 (override));
sprang4847ae62017-06-27 07:06:52 -0700568};
569
philipel9b058032020-02-10 11:30:00 +0100570class MockEncoderSelector
571 : public VideoEncoderFactory::EncoderSelectorInterface {
572 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200573 MOCK_METHOD(void,
574 OnCurrentEncoder,
575 (const SdpVideoFormat& format),
576 (override));
577 MOCK_METHOD(absl::optional<SdpVideoFormat>,
578 OnAvailableBitrate,
579 (const DataRate& rate),
580 (override));
581 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100582};
583
perkj803d97f2016-11-01 11:45:46 -0700584} // namespace
585
mflodmancc3d4422017-08-03 08:27:51 -0700586class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700587 public:
588 static const int kDefaultTimeoutMs = 30 * 1000;
589
mflodmancc3d4422017-08-03 08:27:51 -0700590 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700591 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700592 codec_width_(320),
593 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200594 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200595 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700596 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200597 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700598 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700599 Clock::GetRealTimeClock(),
600 video_send_config_,
601 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700602 sink_(&fake_encoder_) {}
603
604 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700605 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700606 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200607 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800608 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200609 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200610 video_send_config_.rtp.payload_name = "FAKE";
611 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700612
Per512ecb32016-09-23 15:52:06 +0200613 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200614 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700615 video_encoder_config.video_stream_factory =
616 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100617 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700618
619 // Framerate limit is specified by the VideoStreamFactory.
620 std::vector<VideoStream> streams =
621 video_encoder_config.video_stream_factory->CreateEncoderStreams(
622 codec_width_, codec_height_, video_encoder_config);
623 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100624 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700625
Niels Möllerf1338562018-04-26 09:51:47 +0200626 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800627 }
628
Niels Möllerf1338562018-04-26 09:51:47 +0200629 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700630 if (video_stream_encoder_)
631 video_stream_encoder_->Stop();
632 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200633 stats_proxy_.get(), video_send_config_.encoder_settings,
634 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700635 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
636 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700637 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700638 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
639 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200640 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800642 }
643
644 void ResetEncoder(const std::string& payload_name,
645 size_t num_streams,
646 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700647 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700648 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200649 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800650
651 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200652 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800653 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100654 video_encoder_config.max_bitrate_bps =
655 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800656 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700657 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
658 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700659 video_encoder_config.content_type =
660 screenshare ? VideoEncoderConfig::ContentType::kScreen
661 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700662 if (payload_name == "VP9") {
663 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
664 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200665 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700666 video_encoder_config.encoder_specific_settings =
667 new rtc::RefCountedObject<
668 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
669 }
Niels Möllerf1338562018-04-26 09:51:47 +0200670 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700671 }
672
sprang57c2fff2017-01-16 06:24:02 -0800673 VideoFrame CreateFrame(int64_t ntp_time_ms,
674 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100675 VideoFrame frame =
676 VideoFrame::Builder()
677 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
678 destruction_event, codec_width_, codec_height_))
679 .set_timestamp_rtp(99)
680 .set_timestamp_ms(99)
681 .set_rotation(kVideoRotation_0)
682 .build();
sprang57c2fff2017-01-16 06:24:02 -0800683 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700684 return frame;
685 }
686
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100687 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
688 rtc::Event* destruction_event,
689 int offset_x) const {
690 VideoFrame frame =
691 VideoFrame::Builder()
692 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
693 destruction_event, codec_width_, codec_height_))
694 .set_timestamp_rtp(99)
695 .set_timestamp_ms(99)
696 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100697 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100698 .build();
699 frame.set_ntp_time_ms(ntp_time_ms);
700 return frame;
701 }
702
sprang57c2fff2017-01-16 06:24:02 -0800703 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100704 VideoFrame frame =
705 VideoFrame::Builder()
706 .set_video_frame_buffer(
707 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
708 .set_timestamp_rtp(99)
709 .set_timestamp_ms(99)
710 .set_rotation(kVideoRotation_0)
711 .build();
sprang57c2fff2017-01-16 06:24:02 -0800712 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700713 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700714 return frame;
715 }
716
Noah Richards51db4212019-06-12 06:59:12 -0700717 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
718 rtc::Event* destruction_event,
719 int width,
720 int height) const {
721 VideoFrame frame =
722 VideoFrame::Builder()
723 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
724 destruction_event, width, height))
725 .set_timestamp_rtp(99)
726 .set_timestamp_ms(99)
727 .set_rotation(kVideoRotation_0)
728 .build();
729 frame.set_ntp_time_ms(ntp_time_ms);
730 return frame;
731 }
732
733 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
734 rtc::Event* destruction_event) const {
735 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
736 codec_height_);
737 }
738
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100739 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
740 MockBitrateObserver bitrate_observer;
741 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
742
743 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
744 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100746 DataRate::BitsPerSec(kTargetBitrateBps),
747 DataRate::BitsPerSec(kTargetBitrateBps),
748 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100749
750 video_source_.IncomingCapturedFrame(
751 CreateFrame(1, codec_width_, codec_height_));
752 WaitForEncodedFrame(1);
753 }
754
sprang4847ae62017-06-27 07:06:52 -0700755 void WaitForEncodedFrame(int64_t expected_ntp_time) {
756 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100757 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700758 }
759
760 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
761 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100762 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700763 return ok;
764 }
765
766 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
767 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100768 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700769 }
770
771 void ExpectDroppedFrame() {
772 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100773 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700774 }
775
776 bool WaitForFrame(int64_t timeout_ms) {
777 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100778 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700779 return ok;
780 }
781
perkj26091b12016-09-01 01:17:40 -0700782 class TestEncoder : public test::FakeEncoder {
783 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100784 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700785
asaperssonfab67072017-04-04 05:51:49 -0700786 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200787 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700788 return config_;
789 }
790
791 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200792 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700793 block_next_encode_ = true;
794 }
795
Erik Språngaed30702018-11-05 12:57:17 +0100796 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200797 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100798 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100799 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100800 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100801 info.scaling_settings = VideoEncoder::ScalingSettings(
802 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100803 }
804 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100805 for (int i = 0; i < kMaxSpatialLayers; ++i) {
806 if (temporal_layers_supported_[i]) {
807 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
808 info.fps_allocation[i].resize(num_layers);
809 }
810 }
Erik Språngaed30702018-11-05 12:57:17 +0100811 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200812
813 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100814 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100815 return info;
kthelgason876222f2016-11-29 01:44:11 -0800816 }
817
Erik Språngb7cb7b52019-02-26 15:52:33 +0100818 int32_t RegisterEncodeCompleteCallback(
819 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200820 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100821 encoded_image_callback_ = callback;
822 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
823 }
824
perkjfa10b552016-10-02 23:45:26 -0700825 void ContinueEncode() { continue_encode_event_.Set(); }
826
827 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
828 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200829 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700830 EXPECT_EQ(timestamp_, timestamp);
831 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
832 }
833
kthelgason2fc52542017-03-03 00:24:41 -0800834 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200835 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800836 quality_scaling_ = b;
837 }
kthelgasonad9010c2017-02-14 00:46:51 -0800838
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100839 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200840 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100841 requested_resolution_alignment_ = requested_resolution_alignment;
842 }
843
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200845 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100846 is_hardware_accelerated_ = is_hardware_accelerated;
847 }
848
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
850 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200851 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100852 temporal_layers_supported_[spatial_idx] = supported;
853 }
854
Sergey Silkin6456e352019-07-08 17:56:40 +0200855 void SetResolutionBitrateLimits(
856 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200857 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200858 resolution_bitrate_limits_ = thresholds;
859 }
860
sprangfe627f32017-03-29 08:24:59 -0700861 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200862 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700863 force_init_encode_failed_ = force_failure;
864 }
865
Niels Möller6bb5ab92019-01-11 11:11:10 +0100866 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200867 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100868 rate_factor_ = rate_factor;
869 }
870
Erik Språngd7329ca2019-02-21 21:19:53 +0100871 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200872 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100873 return last_framerate_;
874 }
875
Erik Språngd7329ca2019-02-21 21:19:53 +0100876 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200877 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100878 return last_update_rect_;
879 }
880
Niels Möller87e2d782019-03-07 10:18:23 +0100881 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200882 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100883 return last_frame_types_;
884 }
885
886 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100887 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100888 keyframe ? VideoFrameType::kVideoFrameKey
889 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100890 {
Markus Handella3765182020-07-08 13:13:32 +0200891 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100892 last_frame_types_ = frame_type;
893 }
Niels Möllerb859b322019-03-07 12:40:01 +0100894 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100895 }
896
Erik Språngb7cb7b52019-02-26 15:52:33 +0100897 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200898 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100899 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
900 }
901
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200902 void SetEncodedImageData(
903 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200904 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200905 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200906 }
907
Erik Språngd7329ca2019-02-21 21:19:53 +0100908 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200909 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100910 expect_null_frame_ = true;
911 }
912
Erik Språng5056af02019-09-02 15:53:11 +0200913 absl::optional<VideoEncoder::RateControlParameters>
914 GetAndResetLastRateControlSettings() {
915 auto settings = last_rate_control_settings_;
916 last_rate_control_settings_.reset();
917 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100918 }
919
Sergey Silkin5ee69672019-07-02 14:18:34 +0200920 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200922 return num_encoder_initializations_;
923 }
924
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200925 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200927 return num_set_rates_;
928 }
929
perkjfa10b552016-10-02 23:45:26 -0700930 private:
perkj26091b12016-09-01 01:17:40 -0700931 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100932 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700933 bool block_encode;
934 {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 if (expect_null_frame_) {
937 EXPECT_EQ(input_image.timestamp(), 0u);
938 EXPECT_EQ(input_image.width(), 1);
939 last_frame_types_ = *frame_types;
940 expect_null_frame_ = false;
941 } else {
942 EXPECT_GT(input_image.timestamp(), timestamp_);
943 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
944 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
945 }
perkj26091b12016-09-01 01:17:40 -0700946
947 timestamp_ = input_image.timestamp();
948 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700949 last_input_width_ = input_image.width();
950 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700951 block_encode = block_next_encode_;
952 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100953 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -0700955 }
Niels Möllerb859b322019-03-07 12:40:01 +0100956 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -0700957 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -0700958 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -0700959 return result;
960 }
961
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200962 std::unique_ptr<RTPFragmentationHeader> EncodeHook(
963 EncodedImage* encoded_image,
964 CodecSpecificInfo* codec_specific) override {
965 {
966 MutexLock lock(&mutex_);
967 codec_specific->codecType = config_.codecType;
968 }
969 MutexLock lock(&local_mutex_);
970 if (encoded_image_data_) {
971 encoded_image->SetEncodedData(encoded_image_data_);
972 if (codec_specific->codecType == kVideoCodecH264) {
973 auto fragmentation = std::make_unique<RTPFragmentationHeader>();
974 fragmentation->VerifyAndAllocateFragmentationHeader(1);
975 fragmentation->fragmentationOffset[0] = 4;
976 fragmentation->fragmentationLength[0] = encoded_image->size() - 4;
977 return fragmentation;
978 }
979 }
980 return nullptr;
981 }
982
sprangfe627f32017-03-29 08:24:59 -0700983 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +0200984 const Settings& settings) override {
985 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200986
Markus Handella3765182020-07-08 13:13:32 +0200987 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100988 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200989
990 ++num_encoder_initializations_;
991
Erik Språng82fad3d2018-03-21 09:57:23 +0100992 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -0700993 // Simulate setting up temporal layers, in order to validate the life
994 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +0100995 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +0200996 frame_buffer_controller_ =
997 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -0700998 }
Erik Språngb7cb7b52019-02-26 15:52:33 +0100999 if (force_init_encode_failed_) {
1000 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001001 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001002 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001003
Erik Språngb7cb7b52019-02-26 15:52:33 +01001004 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001005 return res;
1006 }
1007
Erik Språngb7cb7b52019-02-26 15:52:33 +01001008 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001009 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001010 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1011 initialized_ = EncoderState::kUninitialized;
1012 return FakeEncoder::Release();
1013 }
1014
Erik Språng16cb8f52019-04-12 13:59:09 +02001015 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001016 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001017 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001018 VideoBitrateAllocation adjusted_rate_allocation;
1019 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1020 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001021 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001022 adjusted_rate_allocation.SetBitrate(
1023 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001024 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001025 rate_factor_));
1026 }
1027 }
1028 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001029 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001030 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001031 RateControlParameters adjusted_paramters = parameters;
1032 adjusted_paramters.bitrate = adjusted_rate_allocation;
1033 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001034 }
1035
Markus Handella3765182020-07-08 13:13:32 +02001036 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001037 enum class EncoderState {
1038 kUninitialized,
1039 kInitializationFailed,
1040 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001041 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1042 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001043 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001044 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1045 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1046 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1047 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1048 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1049 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
1050 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001051 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1052 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001053 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001054 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001055 absl::optional<bool>
1056 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001057 local_mutex_);
1058 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1059 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1060 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001061 absl::optional<VideoEncoder::RateControlParameters>
1062 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001063 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1064 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001065 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001066 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001067 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1068 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001069 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001070 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001071 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001072 RTC_GUARDED_BY(local_mutex_);
1073 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001074 };
1075
mflodmancc3d4422017-08-03 08:27:51 -07001076 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001077 public:
1078 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001079 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001080
perkj26091b12016-09-01 01:17:40 -07001081 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001082 EXPECT_TRUE(
1083 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1084 }
1085
1086 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1087 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001088 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001089 if (!encoded_frame_event_.Wait(timeout_ms))
1090 return false;
perkj26091b12016-09-01 01:17:40 -07001091 {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001093 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001094 }
1095 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001096 return true;
perkj26091b12016-09-01 01:17:40 -07001097 }
1098
sprangb1ca0732017-02-01 08:38:12 -08001099 void WaitForEncodedFrame(uint32_t expected_width,
1100 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001101 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001102 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001103 }
1104
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001105 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001106 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001107 uint32_t width = 0;
1108 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001109 {
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001111 width = last_width_;
1112 height = last_height_;
1113 }
1114 EXPECT_EQ(expected_height, height);
1115 EXPECT_EQ(expected_width, width);
1116 }
1117
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001118 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1119 int width = 0;
1120 int height = 0;
1121 {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001123 width = last_width_;
1124 height = last_height_;
1125 }
1126 EXPECT_EQ(width % resolution_alignment, 0);
1127 EXPECT_EQ(height % resolution_alignment, 0);
1128 }
1129
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001130 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1131 VideoRotation rotation;
1132 {
Markus Handella3765182020-07-08 13:13:32 +02001133 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001134 rotation = last_rotation_;
1135 }
1136 EXPECT_EQ(expected_rotation, rotation);
1137 }
1138
kthelgason2fc52542017-03-03 00:24:41 -08001139 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001140
sprangc5d62e22017-04-02 23:53:04 -07001141 bool WaitForFrame(int64_t timeout_ms) {
1142 return encoded_frame_event_.Wait(timeout_ms);
1143 }
1144
perkj26091b12016-09-01 01:17:40 -07001145 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001147 expect_frames_ = false;
1148 }
1149
asaperssonfab67072017-04-04 05:51:49 -07001150 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001151 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001152 return number_of_reconfigurations_;
1153 }
1154
asaperssonfab67072017-04-04 05:51:49 -07001155 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001156 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001157 return min_transmit_bitrate_bps_;
1158 }
1159
Erik Språngd7329ca2019-02-21 21:19:53 +01001160 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001162 num_expected_layers_ = num_layers;
1163 }
1164
Erik Språngb7cb7b52019-02-26 15:52:33 +01001165 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001166 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001167 return last_capture_time_ms_;
1168 }
1169
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001170 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001171 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001172 return std::move(last_encoded_image_data_);
1173 }
1174
perkj26091b12016-09-01 01:17:40 -07001175 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001176 Result OnEncodedImage(
1177 const EncodedImage& encoded_image,
1178 const CodecSpecificInfo* codec_specific_info,
Danil Chapovalov6d008a82020-07-22 19:49:36 +02001179 const RTPFragmentationHeader* /*fragmentation*/) override {
Markus Handella3765182020-07-08 13:13:32 +02001180 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001181 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001182 last_encoded_image_data_ = std::vector<uint8_t>(
1183 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001184 uint32_t timestamp = encoded_image.Timestamp();
1185 if (last_timestamp_ != timestamp) {
1186 num_received_layers_ = 1;
1187 } else {
1188 ++num_received_layers_;
1189 }
1190 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001191 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001192 last_width_ = encoded_image._encodedWidth;
1193 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001194 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001195 if (num_received_layers_ == num_expected_layers_) {
1196 encoded_frame_event_.Set();
1197 }
sprangb1ca0732017-02-01 08:38:12 -08001198 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001199 }
1200
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001201 void OnEncoderConfigurationChanged(
1202 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001203 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001204 VideoEncoderConfig::ContentType content_type,
1205 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001206 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001207 ++number_of_reconfigurations_;
1208 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1209 }
1210
Markus Handella3765182020-07-08 13:13:32 +02001211 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001212 TestEncoder* test_encoder_;
1213 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001214 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001215 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001216 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001217 uint32_t last_height_ = 0;
1218 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001219 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001220 size_t num_expected_layers_ = 1;
1221 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001222 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001223 int number_of_reconfigurations_ = 0;
1224 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001225 };
1226
Sergey Silkin5ee69672019-07-02 14:18:34 +02001227 class VideoBitrateAllocatorProxyFactory
1228 : public VideoBitrateAllocatorFactory {
1229 public:
1230 VideoBitrateAllocatorProxyFactory()
1231 : bitrate_allocator_factory_(
1232 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1233
1234 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1235 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001236 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001237 codec_config_ = codec;
1238 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1239 }
1240
1241 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001242 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001243 return codec_config_;
1244 }
1245
1246 private:
1247 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1248
Markus Handella3765182020-07-08 13:13:32 +02001249 mutable Mutex mutex_;
1250 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001251 };
1252
perkj26091b12016-09-01 01:17:40 -07001253 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001254 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001255 int codec_width_;
1256 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001257 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001258 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001259 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001260 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001261 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001262 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001263 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001264 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001265 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001266 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001267};
1268
mflodmancc3d4422017-08-03 08:27:51 -07001269TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001270 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001271 DataRate::BitsPerSec(kTargetBitrateBps),
1272 DataRate::BitsPerSec(kTargetBitrateBps),
1273 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001274 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001275 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001276 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001277 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001278 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001279}
1280
mflodmancc3d4422017-08-03 08:27:51 -07001281TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001282 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001283 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001284 // The encoder will cache up to one frame for a short duration. Adding two
1285 // frames means that the first frame will be dropped and the second frame will
1286 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001287 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001288 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001289 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001290
Henrik Boström381d1092020-05-12 18:49:07 +02001291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001292 DataRate::BitsPerSec(kTargetBitrateBps),
1293 DataRate::BitsPerSec(kTargetBitrateBps),
1294 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001295
Sebastian Janssona3177052018-04-10 13:05:49 +02001296 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001297 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001298 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1299
1300 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001301 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001302}
1303
mflodmancc3d4422017-08-03 08:27:51 -07001304TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001306 DataRate::BitsPerSec(kTargetBitrateBps),
1307 DataRate::BitsPerSec(kTargetBitrateBps),
1308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001309 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001310 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001311
Henrik Boström381d1092020-05-12 18:49:07 +02001312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1313 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1314 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001315 // The encoder will cache up to one frame for a short duration. Adding two
1316 // frames means that the first frame will be dropped and the second frame will
1317 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001318 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001319 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001320
Henrik Boström381d1092020-05-12 18:49:07 +02001321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001322 DataRate::BitsPerSec(kTargetBitrateBps),
1323 DataRate::BitsPerSec(kTargetBitrateBps),
1324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001325 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001326 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1327 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001328 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001329}
1330
mflodmancc3d4422017-08-03 08:27:51 -07001331TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001333 DataRate::BitsPerSec(kTargetBitrateBps),
1334 DataRate::BitsPerSec(kTargetBitrateBps),
1335 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001336 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001337 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001338
1339 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001340 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001341
perkja49cbd32016-09-16 07:53:41 -07001342 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001343 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001344 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001345}
1346
mflodmancc3d4422017-08-03 08:27:51 -07001347TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001349 DataRate::BitsPerSec(kTargetBitrateBps),
1350 DataRate::BitsPerSec(kTargetBitrateBps),
1351 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001352
perkja49cbd32016-09-16 07:53:41 -07001353 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001354 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001355
mflodmancc3d4422017-08-03 08:27:51 -07001356 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001357 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001358 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001359 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1360 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001361}
1362
mflodmancc3d4422017-08-03 08:27:51 -07001363TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001365 DataRate::BitsPerSec(kTargetBitrateBps),
1366 DataRate::BitsPerSec(kTargetBitrateBps),
1367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001368
1369 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001370 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001371 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001372 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1373 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001374 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1375 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001376 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001377 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001378
mflodmancc3d4422017-08-03 08:27:51 -07001379 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001380}
1381
Noah Richards51db4212019-06-12 06:59:12 -07001382TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001384 DataRate::BitsPerSec(kTargetBitrateBps),
1385 DataRate::BitsPerSec(kTargetBitrateBps),
1386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001387
1388 rtc::Event frame_destroyed_event;
1389 video_source_.IncomingCapturedFrame(
1390 CreateFakeNativeFrame(1, &frame_destroyed_event));
1391 ExpectDroppedFrame();
1392 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1393 video_stream_encoder_->Stop();
1394}
1395
1396TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1397 // Use the cropping factory.
1398 video_encoder_config_.video_stream_factory =
1399 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1400 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1401 kMaxPayloadLength);
1402 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1403
1404 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001406 DataRate::BitsPerSec(kTargetBitrateBps),
1407 DataRate::BitsPerSec(kTargetBitrateBps),
1408 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001409 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1410 WaitForEncodedFrame(1);
1411 // The encoder will have been configured once.
1412 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1413 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1414 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1415
1416 // Now send in a fake frame that needs to be cropped as the width/height
1417 // aren't divisible by 4 (see CreateEncoderStreams above).
1418 rtc::Event frame_destroyed_event;
1419 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1420 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1421 ExpectDroppedFrame();
1422 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1423 video_stream_encoder_->Stop();
1424}
1425
Ying Wang9b881ab2020-02-07 14:29:32 +01001426TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001428 DataRate::BitsPerSec(kTargetBitrateBps),
1429 DataRate::BitsPerSec(kTargetBitrateBps),
1430 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001431 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1432 WaitForEncodedFrame(1);
1433
Henrik Boström381d1092020-05-12 18:49:07 +02001434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001435 DataRate::BitsPerSec(kTargetBitrateBps),
1436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001438 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1439 // frames. Adding two frames means that the first frame will be dropped and
1440 // the second frame will be sent to the encoder.
1441 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1442 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1443 WaitForEncodedFrame(3);
1444 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1445 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1446 WaitForEncodedFrame(5);
1447 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1448 video_stream_encoder_->Stop();
1449}
1450
mflodmancc3d4422017-08-03 08:27:51 -07001451TEST_F(VideoStreamEncoderTest,
1452 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001454 DataRate::BitsPerSec(kTargetBitrateBps),
1455 DataRate::BitsPerSec(kTargetBitrateBps),
1456 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001457 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001458
1459 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001460 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001462 // The encoder will have been configured once when the first frame is
1463 // received.
1464 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001465
1466 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001467 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001468 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001469 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001470 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001471
1472 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001474 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001475 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001476 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001477
mflodmancc3d4422017-08-03 08:27:51 -07001478 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001479}
1480
mflodmancc3d4422017-08-03 08:27:51 -07001481TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001483 DataRate::BitsPerSec(kTargetBitrateBps),
1484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001486
1487 // Capture a frame and wait for it to synchronize with the encoder thread.
1488 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001489 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001490 // The encoder will have been configured once.
1491 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001492 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1493 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1494
1495 codec_width_ *= 2;
1496 codec_height_ *= 2;
1497 // Capture a frame with a higher resolution and wait for it to synchronize
1498 // with the encoder thread.
1499 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001500 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001501 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1502 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001503 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001504
mflodmancc3d4422017-08-03 08:27:51 -07001505 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001506}
1507
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001508TEST_F(VideoStreamEncoderTest,
1509 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001511 DataRate::BitsPerSec(kTargetBitrateBps),
1512 DataRate::BitsPerSec(kTargetBitrateBps),
1513 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001514
1515 // Capture a frame and wait for it to synchronize with the encoder thread.
1516 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1517 WaitForEncodedFrame(1);
1518
1519 VideoEncoderConfig video_encoder_config;
1520 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1521 // Changing the max payload data length recreates encoder.
1522 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1523 kMaxPayloadLength / 2);
1524
1525 // Capture a frame and wait for it to synchronize with the encoder thread.
1526 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1527 WaitForEncodedFrame(2);
1528 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1529
1530 video_stream_encoder_->Stop();
1531}
1532
Sergey Silkin5ee69672019-07-02 14:18:34 +02001533TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001538
1539 VideoEncoderConfig video_encoder_config;
1540 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1541 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1542 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1543 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1544 kMaxPayloadLength);
1545
1546 // Capture a frame and wait for it to synchronize with the encoder thread.
1547 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1548 WaitForEncodedFrame(1);
1549 // The encoder will have been configured once when the first frame is
1550 // received.
1551 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1552 EXPECT_EQ(kTargetBitrateBps,
1553 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1554 EXPECT_EQ(kStartBitrateBps,
1555 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1556
Sergey Silkin6456e352019-07-08 17:56:40 +02001557 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1558 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001559 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1560 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1561 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1562 kMaxPayloadLength);
1563
1564 // Capture a frame and wait for it to synchronize with the encoder thread.
1565 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1566 WaitForEncodedFrame(2);
1567 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1568 // Bitrate limits have changed - rate allocator should be reconfigured,
1569 // encoder should not be reconfigured.
1570 EXPECT_EQ(kTargetBitrateBps * 2,
1571 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1572 EXPECT_EQ(kStartBitrateBps * 2,
1573 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1574 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1575
1576 video_stream_encoder_->Stop();
1577}
1578
Sergey Silkin6456e352019-07-08 17:56:40 +02001579TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001580 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001585
Sergey Silkincd02eba2020-01-20 14:48:40 +01001586 const uint32_t kMinEncBitrateKbps = 100;
1587 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001588 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001589 /*frame_size_pixels=*/codec_width_ * codec_height_,
1590 /*min_start_bitrate_bps=*/0,
1591 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1592 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001593 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1594
Sergey Silkincd02eba2020-01-20 14:48:40 +01001595 VideoEncoderConfig video_encoder_config;
1596 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1597 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1598 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1599 (kMinEncBitrateKbps + 1) * 1000;
1600 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1601 kMaxPayloadLength);
1602
1603 // When both encoder and app provide bitrate limits, the intersection of
1604 // provided sets should be used.
1605 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1606 WaitForEncodedFrame(1);
1607 EXPECT_EQ(kMaxEncBitrateKbps,
1608 bitrate_allocator_factory_.codec_config().maxBitrate);
1609 EXPECT_EQ(kMinEncBitrateKbps + 1,
1610 bitrate_allocator_factory_.codec_config().minBitrate);
1611
1612 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1613 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1614 (kMinEncBitrateKbps - 1) * 1000;
1615 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1616 kMaxPayloadLength);
1617 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001618 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001619 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001620 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001621 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001622 bitrate_allocator_factory_.codec_config().minBitrate);
1623
Sergey Silkincd02eba2020-01-20 14:48:40 +01001624 video_stream_encoder_->Stop();
1625}
1626
1627TEST_F(VideoStreamEncoderTest,
1628 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001630 DataRate::BitsPerSec(kTargetBitrateBps),
1631 DataRate::BitsPerSec(kTargetBitrateBps),
1632 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001633
1634 const uint32_t kMinAppBitrateKbps = 100;
1635 const uint32_t kMaxAppBitrateKbps = 200;
1636 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1637 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1638 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1639 /*frame_size_pixels=*/codec_width_ * codec_height_,
1640 /*min_start_bitrate_bps=*/0,
1641 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1642 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1643 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1644
1645 VideoEncoderConfig video_encoder_config;
1646 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1647 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1648 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1649 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1651 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001652
Sergey Silkincd02eba2020-01-20 14:48:40 +01001653 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1654 WaitForEncodedFrame(1);
1655 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001656 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001657 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001658 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001659
1660 video_stream_encoder_->Stop();
1661}
1662
1663TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001664 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001666 DataRate::BitsPerSec(kTargetBitrateBps),
1667 DataRate::BitsPerSec(kTargetBitrateBps),
1668 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001669
1670 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001671 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001672 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001673 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001674 fake_encoder_.SetResolutionBitrateLimits(
1675 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1676
1677 VideoEncoderConfig video_encoder_config;
1678 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1679 video_encoder_config.max_bitrate_bps = 0;
1680 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1681 kMaxPayloadLength);
1682
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001683 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001684 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1685 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001686 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1687 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001688 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1689 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1690
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001691 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001692 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1693 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001694 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1695 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001696 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1697 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1698
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001699 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001700 // encoder for 360p should be used.
1701 video_source_.IncomingCapturedFrame(
1702 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1703 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001704 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1705 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001706 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1707 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1708
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001709 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001710 // ignored.
1711 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1712 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001713 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1714 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001715 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1716 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001717 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1718 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001719 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1720 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1721
1722 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1723 // for 270p should be used.
1724 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1725 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001726 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1727 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001728 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1729 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1730
1731 video_stream_encoder_->Stop();
1732}
1733
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001734TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps),
1738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001739
1740 VideoEncoderConfig video_encoder_config;
1741 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1742 video_encoder_config.max_bitrate_bps = 0;
1743 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1744 kMaxPayloadLength);
1745
1746 // Encode 720p frame to get the default encoder target bitrate.
1747 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1748 WaitForEncodedFrame(1);
1749 const uint32_t kDefaultTargetBitrateFor720pKbps =
1750 bitrate_allocator_factory_.codec_config()
1751 .simulcastStream[0]
1752 .targetBitrate;
1753
1754 // Set the max recommended encoder bitrate to something lower than the default
1755 // target bitrate.
1756 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1757 1280 * 720, 10 * 1000, 10 * 1000,
1758 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1759 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1760
1761 // Change resolution to trigger encoder reinitialization.
1762 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1763 WaitForEncodedFrame(2);
1764 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1765 WaitForEncodedFrame(3);
1766
1767 // Ensure the target bitrate is capped by the max bitrate.
1768 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1769 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1770 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1771 .simulcastStream[0]
1772 .targetBitrate *
1773 1000,
1774 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1775
1776 video_stream_encoder_->Stop();
1777}
1778
mflodmancc3d4422017-08-03 08:27:51 -07001779TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001780 EXPECT_TRUE(video_source_.has_sinks());
1781 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001782 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001783 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001784 EXPECT_FALSE(video_source_.has_sinks());
1785 EXPECT_TRUE(new_video_source.has_sinks());
1786
mflodmancc3d4422017-08-03 08:27:51 -07001787 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001788}
1789
mflodmancc3d4422017-08-03 08:27:51 -07001790TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001791 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001792 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001793 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001794 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001795}
1796
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001797TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1798 constexpr int kRequestedResolutionAlignment = 7;
1799 video_source_.set_adaptation_enabled(true);
1800 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001802 DataRate::BitsPerSec(kTargetBitrateBps),
1803 DataRate::BitsPerSec(kTargetBitrateBps),
1804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001805
1806 // On the 1st frame, we should have initialized the encoder and
1807 // asked for its resolution requirements.
1808 video_source_.IncomingCapturedFrame(
1809 CreateFrame(1, codec_width_, codec_height_));
1810 WaitForEncodedFrame(1);
1811 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1812 kRequestedResolutionAlignment);
1813
1814 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1815 // (It's up the to the encoder to potentially drop the previous frame,
1816 // to avoid coding back-to-back keyframes.)
1817 video_source_.IncomingCapturedFrame(
1818 CreateFrame(2, codec_width_, codec_height_));
1819 WaitForEncodedFrame(2);
1820 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1821
1822 video_stream_encoder_->Stop();
1823}
1824
Jonathan Yubc771b72017-12-08 17:04:29 -08001825TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1826 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001827 const int kWidth = 1280;
1828 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001829
1830 // We rely on the automatic resolution adaptation, but we handle framerate
1831 // adaptation manually by mocking the stats proxy.
1832 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001834 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001836 DataRate::BitsPerSec(kTargetBitrateBps),
1837 DataRate::BitsPerSec(kTargetBitrateBps),
1838 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001839 video_stream_encoder_->SetSource(&video_source_,
1840 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001841 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1845
Jonathan Yubc771b72017-12-08 17:04:29 -08001846 // Adapt down as far as possible.
1847 rtc::VideoSinkWants last_wants;
1848 int64_t t = 1;
1849 int loop_count = 0;
1850 do {
1851 ++loop_count;
1852 last_wants = video_source_.sink_wants();
1853
1854 // Simulate the framerate we've been asked to adapt to.
1855 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1856 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1857 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1858 mock_stats.input_frame_rate = fps;
1859 stats_proxy_->SetMockStats(mock_stats);
1860
1861 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1862 sink_.WaitForEncodedFrame(t);
1863 t += frame_interval_ms;
1864
mflodmancc3d4422017-08-03 08:27:51 -07001865 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001866 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001867 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001868 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1869 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001870 } while (video_source_.sink_wants().max_pixel_count <
1871 last_wants.max_pixel_count ||
1872 video_source_.sink_wants().max_framerate_fps <
1873 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001874
Jonathan Yubc771b72017-12-08 17:04:29 -08001875 // Verify that we've adapted all the way down.
1876 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001877 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001878 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1879 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001880 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001881 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1882 *video_source_.last_sent_height());
1883 EXPECT_EQ(kMinBalancedFramerateFps,
1884 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001885
Jonathan Yubc771b72017-12-08 17:04:29 -08001886 // Adapt back up the same number of times we adapted down.
1887 for (int i = 0; i < loop_count - 1; ++i) {
1888 last_wants = video_source_.sink_wants();
1889
1890 // Simulate the framerate we've been asked to adapt to.
1891 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1892 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1893 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1894 mock_stats.input_frame_rate = fps;
1895 stats_proxy_->SetMockStats(mock_stats);
1896
1897 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1898 sink_.WaitForEncodedFrame(t);
1899 t += frame_interval_ms;
1900
Henrik Boström91aa7322020-04-28 12:24:33 +02001901 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001902 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001903 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001904 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1905 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001906 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1907 last_wants.max_pixel_count ||
1908 video_source_.sink_wants().max_framerate_fps >
1909 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001910 }
1911
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001912 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001913 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001914 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001915 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1916 EXPECT_EQ((loop_count - 1) * 2,
1917 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001918
mflodmancc3d4422017-08-03 08:27:51 -07001919 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001920}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001921
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001922TEST_F(VideoStreamEncoderTest,
1923 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1924 video_stream_encoder_->OnBitrateUpdated(
1925 DataRate::BitsPerSec(kTargetBitrateBps),
1926 DataRate::BitsPerSec(kTargetBitrateBps),
1927 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001928 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001929
1930 const int kFrameWidth = 1280;
1931 const int kFrameHeight = 720;
1932
1933 int64_t ntp_time = kFrameIntervalMs;
1934
1935 // Force an input frame rate to be available, or the adaptation call won't
1936 // know what framerate to adapt form.
1937 const int kInputFps = 30;
1938 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1939 stats.input_frame_rate = kInputFps;
1940 stats_proxy_->SetMockStats(stats);
1941
1942 video_source_.set_adaptation_enabled(true);
1943 video_stream_encoder_->SetSource(
1944 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001945 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001946 video_source_.IncomingCapturedFrame(
1947 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1948 sink_.WaitForEncodedFrame(ntp_time);
1949 ntp_time += kFrameIntervalMs;
1950
1951 // Trigger CPU overuse.
1952 video_stream_encoder_->TriggerCpuOveruse();
1953 video_source_.IncomingCapturedFrame(
1954 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1955 sink_.WaitForEncodedFrame(ntp_time);
1956 ntp_time += kFrameIntervalMs;
1957
1958 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1959 EXPECT_EQ(std::numeric_limits<int>::max(),
1960 video_source_.sink_wants().max_pixel_count);
1961 // Some framerate constraint should be set.
1962 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1963 EXPECT_LT(restricted_fps, kInputFps);
1964 video_source_.IncomingCapturedFrame(
1965 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1966 sink_.WaitForEncodedFrame(ntp_time);
1967 ntp_time += 100;
1968
Henrik Boström2671dac2020-05-19 16:29:09 +02001969 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001970 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
1971 // Give the encoder queue time to process the change in degradation preference
1972 // by waiting for an encoded frame.
1973 video_source_.IncomingCapturedFrame(
1974 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1975 sink_.WaitForEncodedFrame(ntp_time);
1976 ntp_time += kFrameIntervalMs;
1977
1978 video_stream_encoder_->TriggerQualityLow();
1979 video_source_.IncomingCapturedFrame(
1980 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1981 sink_.WaitForEncodedFrame(ntp_time);
1982 ntp_time += kFrameIntervalMs;
1983
1984 // Some resolution constraint should be set.
1985 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1986 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
1987 kFrameWidth * kFrameHeight);
1988 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
1989
1990 int pixel_count = video_source_.sink_wants().max_pixel_count;
1991 // Triggering a CPU underuse should not change the sink wants since it has
1992 // not been overused for resolution since we changed degradation preference.
1993 video_stream_encoder_->TriggerCpuUnderuse();
1994 video_source_.IncomingCapturedFrame(
1995 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1996 sink_.WaitForEncodedFrame(ntp_time);
1997 ntp_time += kFrameIntervalMs;
1998 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
1999 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2000
Evan Shrubsole64469032020-06-11 10:45:29 +02002001 // Change the degradation preference back. CPU underuse should not adapt since
2002 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002003 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002004 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2005 video_source_.IncomingCapturedFrame(
2006 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2007 sink_.WaitForEncodedFrame(ntp_time);
2008 ntp_time += 100;
2009 // Resolution adaptations is gone after changing degradation preference.
2010 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2011 EXPECT_EQ(std::numeric_limits<int>::max(),
2012 video_source_.sink_wants().max_pixel_count);
2013 // The fps adaptation from above is now back.
2014 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2015
2016 // Trigger CPU underuse.
2017 video_stream_encoder_->TriggerCpuUnderuse();
2018 video_source_.IncomingCapturedFrame(
2019 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2020 sink_.WaitForEncodedFrame(ntp_time);
2021 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002022 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2023
2024 // Trigger QP underuse, fps should return to normal.
2025 video_stream_encoder_->TriggerQualityHigh();
2026 video_source_.IncomingCapturedFrame(
2027 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2028 sink_.WaitForEncodedFrame(ntp_time);
2029 ntp_time += kFrameIntervalMs;
2030 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002031
2032 video_stream_encoder_->Stop();
2033}
2034
mflodmancc3d4422017-08-03 08:27:51 -07002035TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002037 DataRate::BitsPerSec(kTargetBitrateBps),
2038 DataRate::BitsPerSec(kTargetBitrateBps),
2039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002040 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002041
sprangc5d62e22017-04-02 23:53:04 -07002042 const int kFrameWidth = 1280;
2043 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002044
Åsa Persson8c1bf952018-09-13 10:42:19 +02002045 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002046
kthelgason5e13d412016-12-01 03:59:51 -08002047 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002048 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002049 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002050 frame_timestamp += kFrameIntervalMs;
2051
perkj803d97f2016-11-01 11:45:46 -07002052 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002053 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002054 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002055 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002056 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002057 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002058
asapersson0944a802017-04-07 00:57:58 -07002059 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002060 // wanted resolution.
2061 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2062 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2063 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002064 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002065
2066 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002067 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002068 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002069 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002070 // Give the encoder queue time to process the change in degradation preference
2071 // by waiting for an encoded frame.
2072 new_video_source.IncomingCapturedFrame(
2073 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2074 sink_.WaitForEncodedFrame(frame_timestamp);
2075 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002076 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002077 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002078
sprangc5d62e22017-04-02 23:53:04 -07002079 // Force an input frame rate to be available, or the adaptation call won't
2080 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002081 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002082 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002083 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002084 stats_proxy_->SetMockStats(stats);
2085
mflodmancc3d4422017-08-03 08:27:51 -07002086 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002087 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002088 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002089 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002090 frame_timestamp += kFrameIntervalMs;
2091
2092 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002093 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002094 EXPECT_EQ(std::numeric_limits<int>::max(),
2095 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002096 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002097
asapersson02465b82017-04-10 01:12:52 -07002098 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002099 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2100 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002101 // Give the encoder queue time to process the change in degradation preference
2102 // by waiting for an encoded frame.
2103 new_video_source.IncomingCapturedFrame(
2104 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2105 sink_.WaitForEncodedFrame(frame_timestamp);
2106 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002107 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002108
mflodmancc3d4422017-08-03 08:27:51 -07002109 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002110 new_video_source.IncomingCapturedFrame(
2111 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002112 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002113 frame_timestamp += kFrameIntervalMs;
2114
2115 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002116 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002117
2118 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002119 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002120 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002121 // Give the encoder queue time to process the change in degradation preference
2122 // by waiting for an encoded frame.
2123 new_video_source.IncomingCapturedFrame(
2124 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2125 sink_.WaitForEncodedFrame(frame_timestamp);
2126 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002127 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2128 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002129 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002130 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002131
2132 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002133 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002134 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002135 // Give the encoder queue time to process the change in degradation preference
2136 // by waiting for an encoded frame.
2137 new_video_source.IncomingCapturedFrame(
2138 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2139 sink_.WaitForEncodedFrame(frame_timestamp);
2140 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002141 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2142 EXPECT_EQ(std::numeric_limits<int>::max(),
2143 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002144 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002145
mflodmancc3d4422017-08-03 08:27:51 -07002146 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002147}
2148
mflodmancc3d4422017-08-03 08:27:51 -07002149TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002151 DataRate::BitsPerSec(kTargetBitrateBps),
2152 DataRate::BitsPerSec(kTargetBitrateBps),
2153 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002154
asaperssonfab67072017-04-04 05:51:49 -07002155 const int kWidth = 1280;
2156 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002157 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002158 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002159 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2160 EXPECT_FALSE(stats.bw_limited_resolution);
2161 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2162
2163 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002164 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002165 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002166 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002167
2168 stats = stats_proxy_->GetStats();
2169 EXPECT_TRUE(stats.bw_limited_resolution);
2170 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2171
2172 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002173 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002174 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002175 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002176
2177 stats = stats_proxy_->GetStats();
2178 EXPECT_FALSE(stats.bw_limited_resolution);
2179 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2180 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2181
mflodmancc3d4422017-08-03 08:27:51 -07002182 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002183}
2184
mflodmancc3d4422017-08-03 08:27:51 -07002185TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002187 DataRate::BitsPerSec(kTargetBitrateBps),
2188 DataRate::BitsPerSec(kTargetBitrateBps),
2189 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002190
2191 const int kWidth = 1280;
2192 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002193 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002194 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002195 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2196 EXPECT_FALSE(stats.cpu_limited_resolution);
2197 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2198
2199 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002200 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002201 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002202 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002203
2204 stats = stats_proxy_->GetStats();
2205 EXPECT_TRUE(stats.cpu_limited_resolution);
2206 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2207
2208 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002209 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002210 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002211 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002212
2213 stats = stats_proxy_->GetStats();
2214 EXPECT_FALSE(stats.cpu_limited_resolution);
2215 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002216 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002217
mflodmancc3d4422017-08-03 08:27:51 -07002218 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002219}
2220
mflodmancc3d4422017-08-03 08:27:51 -07002221TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002222 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002223 DataRate::BitsPerSec(kTargetBitrateBps),
2224 DataRate::BitsPerSec(kTargetBitrateBps),
2225 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002226
asaperssonfab67072017-04-04 05:51:49 -07002227 const int kWidth = 1280;
2228 const int kHeight = 720;
2229 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002230 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002231 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002232 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002233 EXPECT_FALSE(stats.cpu_limited_resolution);
2234 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2235
asaperssonfab67072017-04-04 05:51:49 -07002236 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002237 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002238 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002239 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002240 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002241 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002242 EXPECT_TRUE(stats.cpu_limited_resolution);
2243 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2244
2245 // Set new source with adaptation still enabled.
2246 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002247 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002248 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002249
asaperssonfab67072017-04-04 05:51:49 -07002250 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002251 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002252 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002253 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002254 EXPECT_TRUE(stats.cpu_limited_resolution);
2255 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2256
2257 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002258 video_stream_encoder_->SetSource(&new_video_source,
2259 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002260
asaperssonfab67072017-04-04 05:51:49 -07002261 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002262 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002263 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002264 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002265 EXPECT_FALSE(stats.cpu_limited_resolution);
2266 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2267
2268 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002270 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002271
asaperssonfab67072017-04-04 05:51:49 -07002272 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002273 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002274 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002275 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002276 EXPECT_TRUE(stats.cpu_limited_resolution);
2277 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2278
asaperssonfab67072017-04-04 05:51:49 -07002279 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002280 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002281 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002282 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002283 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002284 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002285 EXPECT_FALSE(stats.cpu_limited_resolution);
2286 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002287 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002288
mflodmancc3d4422017-08-03 08:27:51 -07002289 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002290}
2291
mflodmancc3d4422017-08-03 08:27:51 -07002292TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002294 DataRate::BitsPerSec(kTargetBitrateBps),
2295 DataRate::BitsPerSec(kTargetBitrateBps),
2296 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002297
asaperssonfab67072017-04-04 05:51:49 -07002298 const int kWidth = 1280;
2299 const int kHeight = 720;
2300 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002301 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002302 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002303 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002304 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002305 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002306
2307 // Set new source with adaptation still enabled.
2308 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002309 video_stream_encoder_->SetSource(&new_video_source,
2310 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002311
asaperssonfab67072017-04-04 05:51:49 -07002312 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002313 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002314 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002315 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002316 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002317 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002318
asaperssonfab67072017-04-04 05:51:49 -07002319 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002320 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002321 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002322 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002323 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002324 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002325 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002326 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002327
asaperssonfab67072017-04-04 05:51:49 -07002328 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002329 video_stream_encoder_->SetSource(&new_video_source,
2330 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002331
asaperssonfab67072017-04-04 05:51:49 -07002332 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002333 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002334 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002335 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002336 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002337 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002338
asapersson02465b82017-04-10 01:12:52 -07002339 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002340 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002341 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002342
asaperssonfab67072017-04-04 05:51:49 -07002343 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002344 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002345 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002346 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002347 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002348 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2349 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002350
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002352}
2353
mflodmancc3d4422017-08-03 08:27:51 -07002354TEST_F(VideoStreamEncoderTest,
2355 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002357 DataRate::BitsPerSec(kTargetBitrateBps),
2358 DataRate::BitsPerSec(kTargetBitrateBps),
2359 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002360
2361 const int kWidth = 1280;
2362 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002363 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002364 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002365 video_source_.IncomingCapturedFrame(
2366 CreateFrame(timestamp_ms, kWidth, kHeight));
2367 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002368 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2369 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2371
2372 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002373 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002374 timestamp_ms += kFrameIntervalMs;
2375 video_source_.IncomingCapturedFrame(
2376 CreateFrame(timestamp_ms, kWidth, kHeight));
2377 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002378 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2380 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2381
2382 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002383 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002384 timestamp_ms += kFrameIntervalMs;
2385 video_source_.IncomingCapturedFrame(
2386 CreateFrame(timestamp_ms, kWidth, kHeight));
2387 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002388 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2391
Niels Möller4db138e2018-04-19 09:04:13 +02002392 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002393 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002394
2395 VideoEncoderConfig video_encoder_config;
2396 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2397 // Make format different, to force recreation of encoder.
2398 video_encoder_config.video_format.parameters["foo"] = "foo";
2399 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002400 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002401 timestamp_ms += kFrameIntervalMs;
2402 video_source_.IncomingCapturedFrame(
2403 CreateFrame(timestamp_ms, kWidth, kHeight));
2404 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002405 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2407 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2408
mflodmancc3d4422017-08-03 08:27:51 -07002409 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002410}
2411
mflodmancc3d4422017-08-03 08:27:51 -07002412TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002413 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002415 DataRate::BitsPerSec(kTargetBitrateBps),
2416 DataRate::BitsPerSec(kTargetBitrateBps),
2417 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2418
2419 const int kWidth = 1280;
2420 const int kHeight = 720;
2421 int sequence = 1;
2422
2423 // Enable BALANCED preference, no initial limitation.
2424 test::FrameForwarder source;
2425 video_stream_encoder_->SetSource(&source,
2426 webrtc::DegradationPreference::BALANCED);
2427 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2428 WaitForEncodedFrame(sequence++);
2429 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2430 EXPECT_FALSE(stats.cpu_limited_resolution);
2431 EXPECT_FALSE(stats.cpu_limited_framerate);
2432 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2433
2434 // Trigger CPU overuse, should now adapt down.
2435 video_stream_encoder_->TriggerCpuOveruse();
2436 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2437 WaitForEncodedFrame(sequence++);
2438 stats = stats_proxy_->GetStats();
2439 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2440
2441 // Set new degradation preference should clear restrictions since we changed
2442 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002443 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002444 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2445 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2446 WaitForEncodedFrame(sequence++);
2447 stats = stats_proxy_->GetStats();
2448 EXPECT_FALSE(stats.cpu_limited_resolution);
2449 EXPECT_FALSE(stats.cpu_limited_framerate);
2450 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2451
2452 // Force an input frame rate to be available, or the adaptation call won't
2453 // know what framerate to adapt from.
2454 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2455 mock_stats.input_frame_rate = 30;
2456 stats_proxy_->SetMockStats(mock_stats);
2457 video_stream_encoder_->TriggerCpuOveruse();
2458 stats_proxy_->ResetMockStats();
2459 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2460 WaitForEncodedFrame(sequence++);
2461
2462 // We have now adapted once.
2463 stats = stats_proxy_->GetStats();
2464 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2465
2466 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002467 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2468 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002469 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2470 WaitForEncodedFrame(sequence++);
2471 stats = stats_proxy_->GetStats();
2472 EXPECT_FALSE(stats.cpu_limited_resolution);
2473 EXPECT_FALSE(stats.cpu_limited_framerate);
2474 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2475
2476 video_stream_encoder_->Stop();
2477}
2478
2479TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002480 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002482 DataRate::BitsPerSec(kTargetBitrateBps),
2483 DataRate::BitsPerSec(kTargetBitrateBps),
2484 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002485
asapersson0944a802017-04-07 00:57:58 -07002486 const int kWidth = 1280;
2487 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002488 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002489
asaperssonfab67072017-04-04 05:51:49 -07002490 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002491 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002492 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002493 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002494 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002495 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2496
asapersson02465b82017-04-10 01:12:52 -07002497 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002498 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002499 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002500 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002501 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002502 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002503 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002504 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2505
2506 // Set new source with adaptation still enabled.
2507 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002508 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002509 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002510
2511 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002512 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002513 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002514 stats = stats_proxy_->GetStats();
2515 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002516 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002517 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2518
sprangc5d62e22017-04-02 23:53:04 -07002519 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002520 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002521 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002522 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002523 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002524 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002525 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002526 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002527 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002528 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002529 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2530
sprangc5d62e22017-04-02 23:53:04 -07002531 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002532 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002533 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2534 mock_stats.input_frame_rate = 30;
2535 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002536 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002537 stats_proxy_->ResetMockStats();
2538
2539 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002540 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002541 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002542
2543 // Framerate now adapted.
2544 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002545 EXPECT_FALSE(stats.cpu_limited_resolution);
2546 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002547 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2548
2549 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002550 video_stream_encoder_->SetSource(&new_video_source,
2551 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002552 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002553 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002554 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002555
2556 stats = stats_proxy_->GetStats();
2557 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002558 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002559 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2560
2561 // Try to trigger overuse. Should not succeed.
2562 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002563 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002564 stats_proxy_->ResetMockStats();
2565
2566 stats = stats_proxy_->GetStats();
2567 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002568 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002569 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2570
2571 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002573 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002574 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002575 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002576 stats = stats_proxy_->GetStats();
2577 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002578 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002579 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002580
2581 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002582 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002583 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002584 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002585 stats = stats_proxy_->GetStats();
2586 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002587 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002588 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2589
2590 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002591 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002592 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002593 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002594 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002595 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002596 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002597 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002598 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002599 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002600 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2601
2602 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002603 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002604 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002605 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002606 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002607 stats = stats_proxy_->GetStats();
2608 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002609 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002610 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002611 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002612
mflodmancc3d4422017-08-03 08:27:51 -07002613 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002614}
2615
mflodmancc3d4422017-08-03 08:27:51 -07002616TEST_F(VideoStreamEncoderTest,
2617 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002618 const int kWidth = 1280;
2619 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002621 DataRate::BitsPerSec(kTargetBitrateBps),
2622 DataRate::BitsPerSec(kTargetBitrateBps),
2623 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002624
asaperssonfab67072017-04-04 05:51:49 -07002625 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002626 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002627
asaperssonfab67072017-04-04 05:51:49 -07002628 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002629 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002630
asaperssonfab67072017-04-04 05:51:49 -07002631 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002633
asaperssonfab67072017-04-04 05:51:49 -07002634 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002635 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002636
kthelgason876222f2016-11-29 01:44:11 -08002637 // Expect a scale down.
2638 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002639 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002640
asapersson02465b82017-04-10 01:12:52 -07002641 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002642 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002643 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002644 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002645
asaperssonfab67072017-04-04 05:51:49 -07002646 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002647 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002648 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002649 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002650
asaperssonfab67072017-04-04 05:51:49 -07002651 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002652 EXPECT_EQ(std::numeric_limits<int>::max(),
2653 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002654
asaperssonfab67072017-04-04 05:51:49 -07002655 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002656 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002657 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002658 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002659
asapersson02465b82017-04-10 01:12:52 -07002660 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002661 EXPECT_EQ(std::numeric_limits<int>::max(),
2662 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002663
mflodmancc3d4422017-08-03 08:27:51 -07002664 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002665}
2666
mflodmancc3d4422017-08-03 08:27:51 -07002667TEST_F(VideoStreamEncoderTest,
2668 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002669 const int kWidth = 1280;
2670 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002672 DataRate::BitsPerSec(kTargetBitrateBps),
2673 DataRate::BitsPerSec(kTargetBitrateBps),
2674 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002675
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002676 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002677 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002678 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002679 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002680
2681 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002682 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2686
2687 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002689 EXPECT_THAT(source.sink_wants(),
2690 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002691 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2692 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2693 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2694
2695 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002697 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2698 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2699 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2700
mflodmancc3d4422017-08-03 08:27:51 -07002701 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002702}
2703
mflodmancc3d4422017-08-03 08:27:51 -07002704TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002705 const int kWidth = 1280;
2706 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002708 DataRate::BitsPerSec(kTargetBitrateBps),
2709 DataRate::BitsPerSec(kTargetBitrateBps),
2710 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002711
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002712 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002713 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002714 video_stream_encoder_->SetSource(&source,
2715 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002716 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2717 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002719
2720 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002721 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002722 EXPECT_THAT(source.sink_wants(),
2723 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002724 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2726 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2727
2728 // Trigger adapt down for same input resolution, expect no change.
2729 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2730 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002731 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002732 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2733 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2734 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2735
2736 // Trigger adapt down for larger input resolution, expect no change.
2737 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2738 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002739 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002740 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2741 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2742 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002745}
2746
mflodmancc3d4422017-08-03 08:27:51 -07002747TEST_F(VideoStreamEncoderTest,
2748 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002749 const int kWidth = 1280;
2750 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002752 DataRate::BitsPerSec(kTargetBitrateBps),
2753 DataRate::BitsPerSec(kTargetBitrateBps),
2754 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002755
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002756 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002757 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002758 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002759 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002760
2761 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002763 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2765 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2766
2767 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002768 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002769 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2771 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2772
mflodmancc3d4422017-08-03 08:27:51 -07002773 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002774}
2775
mflodmancc3d4422017-08-03 08:27:51 -07002776TEST_F(VideoStreamEncoderTest,
2777 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002778 const int kWidth = 1280;
2779 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002781 DataRate::BitsPerSec(kTargetBitrateBps),
2782 DataRate::BitsPerSec(kTargetBitrateBps),
2783 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002784
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002785 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002786 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002787 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002788 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002789
2790 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002791 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002792 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002794 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2795
2796 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002797 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2801
mflodmancc3d4422017-08-03 08:27:51 -07002802 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002803}
2804
mflodmancc3d4422017-08-03 08:27:51 -07002805TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002806 const int kWidth = 1280;
2807 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002809 DataRate::BitsPerSec(kTargetBitrateBps),
2810 DataRate::BitsPerSec(kTargetBitrateBps),
2811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002812
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002813 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002814 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 video_stream_encoder_->SetSource(&source,
2816 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002817
2818 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2819 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002820 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2822 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2824
2825 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002827 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2829 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2831
mflodmancc3d4422017-08-03 08:27:51 -07002832 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002833}
2834
mflodmancc3d4422017-08-03 08:27:51 -07002835TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002836 const int kWidth = 1280;
2837 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002839 DataRate::BitsPerSec(kTargetBitrateBps),
2840 DataRate::BitsPerSec(kTargetBitrateBps),
2841 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002843 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002844 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002845 video_stream_encoder_->SetSource(&source,
2846 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002847
2848 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2849 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002850 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2853 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2854
2855 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002856 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002857 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002858 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2859 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2860 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2861
mflodmancc3d4422017-08-03 08:27:51 -07002862 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002863}
2864
mflodmancc3d4422017-08-03 08:27:51 -07002865TEST_F(VideoStreamEncoderTest,
2866 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002867 const int kWidth = 1280;
2868 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002870 DataRate::BitsPerSec(kTargetBitrateBps),
2871 DataRate::BitsPerSec(kTargetBitrateBps),
2872 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002873
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002874 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002875 AdaptingFrameForwarder source;
2876 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002878 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002879
2880 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002881 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002882 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2884 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2885
2886 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002887 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002888 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002889 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002890 EXPECT_THAT(source.sink_wants(),
2891 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002892 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2894
2895 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002896 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002897 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002898 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2899 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2900 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2901
mflodmancc3d4422017-08-03 08:27:51 -07002902 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002903}
2904
mflodmancc3d4422017-08-03 08:27:51 -07002905TEST_F(VideoStreamEncoderTest,
2906 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002907 const int kWidth = 1280;
2908 const int kHeight = 720;
2909 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002911 DataRate::BitsPerSec(kTargetBitrateBps),
2912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002914
2915 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2916 stats.input_frame_rate = kInputFps;
2917 stats_proxy_->SetMockStats(stats);
2918
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002919 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002920 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2921 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002922 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002923
2924 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002925 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002926 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2927 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002928 EXPECT_THAT(video_source_.sink_wants(),
2929 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002930
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002931 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002932 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002933 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002934 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002935 // Give the encoder queue time to process the change in degradation preference
2936 // by waiting for an encoded frame.
2937 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2938 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002939 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002940
2941 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002943 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2944 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002945 EXPECT_THAT(new_video_source.sink_wants(),
2946 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002947
2948 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002949 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002950 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002951
mflodmancc3d4422017-08-03 08:27:51 -07002952 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002953}
2954
mflodmancc3d4422017-08-03 08:27:51 -07002955TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002956 const int kWidth = 1280;
2957 const int kHeight = 720;
2958 const size_t kNumFrames = 10;
2959
Henrik Boström381d1092020-05-12 18:49:07 +02002960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002961 DataRate::BitsPerSec(kTargetBitrateBps),
2962 DataRate::BitsPerSec(kTargetBitrateBps),
2963 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002964
asaperssond0de2952017-04-21 01:47:31 -07002965 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002966 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002967 video_source_.set_adaptation_enabled(true);
2968
2969 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2970 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2971
2972 int downscales = 0;
2973 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02002974 video_source_.IncomingCapturedFrame(
2975 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
2976 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07002977
asaperssonfab67072017-04-04 05:51:49 -07002978 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07002979 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07002980 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07002981 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07002982
2983 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
2984 ++downscales;
2985
2986 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2987 EXPECT_EQ(downscales,
2988 stats_proxy_->GetStats().number_of_quality_adapt_changes);
2989 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002990 }
mflodmancc3d4422017-08-03 08:27:51 -07002991 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07002992}
2993
mflodmancc3d4422017-08-03 08:27:51 -07002994TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07002995 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
2996 const int kWidth = 1280;
2997 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002999 DataRate::BitsPerSec(kTargetBitrateBps),
3000 DataRate::BitsPerSec(kTargetBitrateBps),
3001 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003002
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003003 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003004 AdaptingFrameForwarder source;
3005 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003007 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003008
Åsa Persson8c1bf952018-09-13 10:42:19 +02003009 int64_t timestamp_ms = kFrameIntervalMs;
3010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003011 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003012 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003013 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3014 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3015
3016 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003018 timestamp_ms += kFrameIntervalMs;
3019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3020 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003021 EXPECT_THAT(source.sink_wants(),
3022 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003023 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3024 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3025
3026 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003027 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003028 timestamp_ms += kFrameIntervalMs;
3029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003030 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003031 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003032 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3033 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3034
3035 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003037 timestamp_ms += kFrameIntervalMs;
3038 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3039 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003040 EXPECT_THAT(source.sink_wants(),
3041 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003042 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3043 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3044
3045 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003046 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003047 timestamp_ms += kFrameIntervalMs;
3048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003049 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003051 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3052 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3053
mflodmancc3d4422017-08-03 08:27:51 -07003054 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003055}
3056
mflodmancc3d4422017-08-03 08:27:51 -07003057TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003058 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3059 const int kWidth = 1280;
3060 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003062 DataRate::BitsPerSec(kTargetBitrateBps),
3063 DataRate::BitsPerSec(kTargetBitrateBps),
3064 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003065
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003066 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003067 AdaptingFrameForwarder source;
3068 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003069 video_stream_encoder_->SetSource(&source,
3070 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003071
Åsa Persson8c1bf952018-09-13 10:42:19 +02003072 int64_t timestamp_ms = kFrameIntervalMs;
3073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003074 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003075 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003076 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3078
3079 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003080 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003081 timestamp_ms += kFrameIntervalMs;
3082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3083 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003084 EXPECT_THAT(source.sink_wants(),
3085 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3087 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3088
3089 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003090 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003091 timestamp_ms += kFrameIntervalMs;
3092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003093 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003094 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003095 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3096 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3097
3098 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003100 timestamp_ms += kFrameIntervalMs;
3101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3102 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003103 EXPECT_THAT(source.sink_wants(),
3104 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003105 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3106 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3107
3108 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003109 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003110 timestamp_ms += kFrameIntervalMs;
3111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003112 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003113 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003114 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3115 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3116
mflodmancc3d4422017-08-03 08:27:51 -07003117 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003118}
3119
Sergey Silkin41c650b2019-10-14 13:12:19 +02003120TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3121 fake_encoder_.SetResolutionBitrateLimits(
3122 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3123
Henrik Boström381d1092020-05-12 18:49:07 +02003124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003125 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3126 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3127 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3128 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003129
3130 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3131 AdaptingFrameForwarder source;
3132 source.set_adaptation_enabled(true);
3133 video_stream_encoder_->SetSource(
3134 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3135
3136 // Insert 720p frame.
3137 int64_t timestamp_ms = kFrameIntervalMs;
3138 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3139 WaitForEncodedFrame(1280, 720);
3140
3141 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003143 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3144 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3145 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3146 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003147 video_stream_encoder_->TriggerQualityLow();
3148
3149 // Insert 720p frame. It should be downscaled and encoded.
3150 timestamp_ms += kFrameIntervalMs;
3151 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3152 WaitForEncodedFrame(960, 540);
3153
3154 // Trigger adapt up. Higher resolution should not be requested duo to lack
3155 // of bitrate.
3156 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003157 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003158
3159 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003161 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3162 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3163 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3164 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003165
3166 // Trigger adapt up. Higher resolution should be requested.
3167 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003168 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003169
3170 video_stream_encoder_->Stop();
3171}
3172
3173TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3174 fake_encoder_.SetResolutionBitrateLimits(
3175 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3176
3177 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003179 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3180 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3181 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3182 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003183
3184 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3185 AdaptingFrameForwarder source;
3186 source.set_adaptation_enabled(true);
3187 video_stream_encoder_->SetSource(
3188 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3189
3190 // Insert 720p frame. It should be dropped and lower resolution should be
3191 // requested.
3192 int64_t timestamp_ms = kFrameIntervalMs;
3193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3194 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003195 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003196
3197 // Insert 720p frame. It should be downscaled and encoded.
3198 timestamp_ms += kFrameIntervalMs;
3199 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3200 WaitForEncodedFrame(960, 540);
3201
3202 video_stream_encoder_->Stop();
3203}
3204
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003205class BalancedDegradationTest : public VideoStreamEncoderTest {
3206 protected:
3207 void SetupTest() {
3208 // Reset encoder for field trials to take effect.
3209 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003210 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003211
3212 // Enable BALANCED preference.
3213 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003214 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3215 }
3216
3217 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003219 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3220 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003221 }
3222
Åsa Persson45b176f2019-09-30 11:19:05 +02003223 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003224 timestamp_ms_ += kFrameIntervalMs;
3225 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003226 }
3227
3228 void InsertFrameAndWaitForEncoded() {
3229 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003230 sink_.WaitForEncodedFrame(timestamp_ms_);
3231 }
3232
3233 const int kWidth = 640; // pixels:640x360=230400
3234 const int kHeight = 360;
3235 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3236 int64_t timestamp_ms_ = 0;
3237 AdaptingFrameForwarder source_;
3238};
3239
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003240TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003241 test::ScopedFieldTrials field_trials(
3242 "WebRTC-Video-BalancedDegradationSettings/"
3243 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3244 SetupTest();
3245
3246 // Force input frame rate.
3247 const int kInputFps = 24;
3248 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3249 stats.input_frame_rate = kInputFps;
3250 stats_proxy_->SetMockStats(stats);
3251
Åsa Persson45b176f2019-09-30 11:19:05 +02003252 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003253 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003254
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003255 // Trigger adapt down, expect scaled down framerate and resolution,
3256 // since Fps diff (input-requested:0) < threshold.
3257 video_stream_encoder_->TriggerQualityLow();
3258 EXPECT_THAT(source_.sink_wants(),
3259 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003260
3261 video_stream_encoder_->Stop();
3262}
3263
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003264TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003265 test::ScopedFieldTrials field_trials(
3266 "WebRTC-Video-BalancedDegradationSettings/"
3267 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3268 SetupTest();
3269
3270 // Force input frame rate.
3271 const int kInputFps = 25;
3272 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3273 stats.input_frame_rate = kInputFps;
3274 stats_proxy_->SetMockStats(stats);
3275
Åsa Persson45b176f2019-09-30 11:19:05 +02003276 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003277 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003278
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003279 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3280 // Fps diff (input-requested:1) == threshold.
3281 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003282 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003283
3284 video_stream_encoder_->Stop();
3285}
3286
3287TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3288 test::ScopedFieldTrials field_trials(
3289 "WebRTC-Video-BalancedDegradationSettings/"
3290 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3291 SetupTest();
3292
3293 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3294
Åsa Persson45b176f2019-09-30 11:19:05 +02003295 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003296 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003297
3298 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3299 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003300 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003301
3302 video_stream_encoder_->Stop();
3303}
3304
Åsa Perssonccfb3402019-09-25 15:13:04 +02003305TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003306 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003307 "WebRTC-Video-BalancedDegradationSettings/"
3308 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003309 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003310
Åsa Persson1b247f12019-08-14 17:26:39 +02003311 const int kMinBitrateBps = 425000;
3312 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003313 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003314
Åsa Persson45b176f2019-09-30 11:19:05 +02003315 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003316 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003317 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3318
3319 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3320 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003323 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3324
3325 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3326 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003327 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003328 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003329 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3330
Åsa Persson30ab0152019-08-27 12:22:33 +02003331 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3332 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003333 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003334 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003335 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003336 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3337
3338 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003340 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003341 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003342
Åsa Persson30ab0152019-08-27 12:22:33 +02003343 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003344 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003345 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003346 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003347 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003348 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3349
3350 video_stream_encoder_->Stop();
3351}
3352
Åsa Perssonccfb3402019-09-25 15:13:04 +02003353TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003354 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3355 test::ScopedFieldTrials field_trials(
3356 "WebRTC-Video-BalancedDegradationSettings/"
3357 "pixels:57600|129600|230400,fps:7|24|24/");
3358 SetupTest();
3359 OnBitrateUpdated(kLowTargetBitrateBps);
3360
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003361 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003362
3363 // Insert frame, expect scaled down:
3364 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3365 InsertFrame();
3366 EXPECT_FALSE(WaitForFrame(1000));
3367 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3368 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3369
3370 // Insert frame, expect scaled down:
3371 // resolution (320x180@24fps).
3372 InsertFrame();
3373 EXPECT_FALSE(WaitForFrame(1000));
3374 EXPECT_LT(source_.sink_wants().max_pixel_count,
3375 source_.last_wants().max_pixel_count);
3376 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3377
3378 // Frame should not be dropped (min pixels per frame reached).
3379 InsertFrameAndWaitForEncoded();
3380
3381 video_stream_encoder_->Stop();
3382}
3383
3384TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003385 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003386 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003387 "WebRTC-Video-BalancedDegradationSettings/"
3388 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003389 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003390
Åsa Persson30ab0152019-08-27 12:22:33 +02003391 const int kResolutionMinBitrateBps = 435000;
3392 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003393 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003394
Åsa Persson45b176f2019-09-30 11:19:05 +02003395 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003396 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003397 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3398
3399 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3400 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003401 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003402 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003403 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3404
3405 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3406 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003407 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003408 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003409 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3410
3411 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3412 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003413 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003414 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
Åsa Persson30ab0152019-08-27 12:22:33 +02003417 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3418 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003419 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003420 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003421 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3424 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003425 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003426 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3427
3428 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003429 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003430 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003431 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003432 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003433 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3434
3435 video_stream_encoder_->Stop();
3436}
3437
Åsa Perssonccfb3402019-09-25 15:13:04 +02003438TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003439 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003440 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003441 "WebRTC-Video-BalancedDegradationSettings/"
3442 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003443 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003444
Åsa Persson30ab0152019-08-27 12:22:33 +02003445 const int kMinBitrateBps = 425000;
3446 const int kTooLowMinBitrateBps = 424000;
3447 const int kResolutionMinBitrateBps = 435000;
3448 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003449 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003450
Åsa Persson45b176f2019-09-30 11:19:05 +02003451 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003452 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003453 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3456 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003457 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003458 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003459 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3462 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003463 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003464 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003465 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3466
3467 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3468 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003469 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003470 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003471 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3474 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003475 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003476 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477
3478 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003479 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003480 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003481 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003482 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3484
3485 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003486 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003487 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003488 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003489 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3490
3491 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003492 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003493 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003494 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003495 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003496 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
Åsa Persson1b247f12019-08-14 17:26:39 +02003498 video_stream_encoder_->Stop();
3499}
3500
mflodmancc3d4422017-08-03 08:27:51 -07003501TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003502 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3503 const int kWidth = 1280;
3504 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003506 DataRate::BitsPerSec(kTargetBitrateBps),
3507 DataRate::BitsPerSec(kTargetBitrateBps),
3508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003509
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003510 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003511 AdaptingFrameForwarder source;
3512 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003513 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003514 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003515
Åsa Persson8c1bf952018-09-13 10:42:19 +02003516 int64_t timestamp_ms = kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003518 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003519 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003520 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3521 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3522 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3523 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3524
3525 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003526 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003527 timestamp_ms += kFrameIntervalMs;
3528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3529 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003530 EXPECT_THAT(source.sink_wants(),
3531 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003532 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3533 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3535 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3536
3537 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003538 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003539 timestamp_ms += kFrameIntervalMs;
3540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3541 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003542 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003543 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3545 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3546 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3547
Jonathan Yubc771b72017-12-08 17:04:29 -08003548 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003549 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003550 timestamp_ms += kFrameIntervalMs;
3551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003556 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3558
Jonathan Yubc771b72017-12-08 17:04:29 -08003559 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003560 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003561 timestamp_ms += kFrameIntervalMs;
3562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003565 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003566 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3568 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3569 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3570
Jonathan Yubc771b72017-12-08 17:04:29 -08003571 // Trigger quality adapt down, expect no change (min resolution reached).
3572 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003573 timestamp_ms += kFrameIntervalMs;
3574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003576 EXPECT_THAT(source.sink_wants(), FpsMax());
3577 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003578 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3581 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582
Evan Shrubsole64469032020-06-11 10:45:29 +02003583 // Trigger quality adapt up, expect upscaled resolution (480x270).
3584 video_stream_encoder_->TriggerQualityHigh();
3585 timestamp_ms += kFrameIntervalMs;
3586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3587 WaitForEncodedFrame(timestamp_ms);
3588 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3589 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3591 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3592 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3593
3594 // Trigger quality and cpu adapt up since both are most limited, expect
3595 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003596 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003597 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003601 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3604 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003606
Evan Shrubsole64469032020-06-11 10:45:29 +02003607 // Trigger quality and cpu adapt up since both are most limited, expect
3608 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003609 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003610 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003611 timestamp_ms += kFrameIntervalMs;
3612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3613 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003614 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003615 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003618 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3619 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003620
Evan Shrubsole64469032020-06-11 10:45:29 +02003621 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3622 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003623 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003627 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003630 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003631 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003632
3633 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003634 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003635 timestamp_ms += kFrameIntervalMs;
3636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003637 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003638 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003639 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003642 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003643 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003644
mflodmancc3d4422017-08-03 08:27:51 -07003645 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003646}
3647
mflodmancc3d4422017-08-03 08:27:51 -07003648TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003649 const int kWidth = 640;
3650 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003651
Henrik Boström381d1092020-05-12 18:49:07 +02003652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003653 DataRate::BitsPerSec(kTargetBitrateBps),
3654 DataRate::BitsPerSec(kTargetBitrateBps),
3655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003656
perkj803d97f2016-11-01 11:45:46 -07003657 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003658 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003659 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003660 }
3661
mflodmancc3d4422017-08-03 08:27:51 -07003662 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003663 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003664 video_source_.IncomingCapturedFrame(CreateFrame(
3665 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003666 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003667 }
3668
mflodmancc3d4422017-08-03 08:27:51 -07003669 video_stream_encoder_->Stop();
3670 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003671 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003672
Ying Wangef3998f2019-12-09 13:06:53 +01003673 EXPECT_METRIC_EQ(
3674 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3675 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003676 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3677}
3678
mflodmancc3d4422017-08-03 08:27:51 -07003679TEST_F(VideoStreamEncoderTest,
3680 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003682 DataRate::BitsPerSec(kTargetBitrateBps),
3683 DataRate::BitsPerSec(kTargetBitrateBps),
3684 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003685 const int kWidth = 640;
3686 const int kHeight = 360;
3687
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003688 video_stream_encoder_->SetSource(&video_source_,
3689 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003690
3691 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3692 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003693 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003694 }
3695
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->Stop();
3697 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003698 stats_proxy_.reset();
3699
3700 EXPECT_EQ(0,
3701 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3702}
3703
mflodmancc3d4422017-08-03 08:27:51 -07003704TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003705 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003706 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003707
3708 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003709 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003710 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003711 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3712 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003713
sprang57c2fff2017-01-16 06:24:02 -08003714 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003715 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003717 DataRate::BitsPerSec(kLowTargetBitrateBps),
3718 DataRate::BitsPerSec(kLowTargetBitrateBps),
3719 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003720
sprang57c2fff2017-01-16 06:24:02 -08003721 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003722 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3723 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003724 VideoBitrateAllocation bitrate_allocation =
3725 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003726 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003727 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003728 // TODO(srte): The use of millisecs here looks like an error, but the tests
3729 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003730 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003731
3732 // Not called on second frame.
3733 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3734 .Times(0);
3735 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003736 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3737 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003738 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003739
3740 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003741 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3742 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003743 const int64_t start_time_ms = rtc::TimeMillis();
3744 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3745 video_source_.IncomingCapturedFrame(
3746 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3747 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003748 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003749 }
3750
3751 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003752 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003753
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003755}
3756
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003757TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3758 // 2 TLs configured, temporal layers supported by encoder.
3759 const int kNumTemporalLayers = 2;
3760 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3761 fake_encoder_.SetTemporalLayersSupported(0, true);
3762
3763 // Bitrate allocated across temporal layers.
3764 const int kTl0Bps = kTargetBitrateBps *
3765 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003766 kNumTemporalLayers, /*temporal_id*/ 0,
3767 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003768 const int kTl1Bps = kTargetBitrateBps *
3769 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003770 kNumTemporalLayers, /*temporal_id*/ 1,
3771 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003772 VideoBitrateAllocation expected_bitrate;
3773 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3774 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3775
3776 VerifyAllocatedBitrate(expected_bitrate);
3777 video_stream_encoder_->Stop();
3778}
3779
3780TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3781 // 2 TLs configured, temporal layers not supported by encoder.
3782 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3783 fake_encoder_.SetTemporalLayersSupported(0, false);
3784
3785 // Temporal layers not supported by the encoder.
3786 // Total bitrate should be at ti:0.
3787 VideoBitrateAllocation expected_bitrate;
3788 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3789
3790 VerifyAllocatedBitrate(expected_bitrate);
3791 video_stream_encoder_->Stop();
3792}
3793
3794TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3795 // 2 TLs configured, temporal layers only supported for first stream.
3796 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3797 fake_encoder_.SetTemporalLayersSupported(0, true);
3798 fake_encoder_.SetTemporalLayersSupported(1, false);
3799
3800 const int kS0Bps = 150000;
3801 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003802 kS0Bps *
3803 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3804 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003805 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003806 kS0Bps *
3807 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3808 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003809 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3810 // Temporal layers not supported by si:1.
3811 VideoBitrateAllocation expected_bitrate;
3812 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3813 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3814 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3815
3816 VerifyAllocatedBitrate(expected_bitrate);
3817 video_stream_encoder_->Stop();
3818}
3819
Niels Möller7dc26b72017-12-06 10:27:48 +01003820TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3821 const int kFrameWidth = 1280;
3822 const int kFrameHeight = 720;
3823 const int kFramerate = 24;
3824
Henrik Boström381d1092020-05-12 18:49:07 +02003825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003826 DataRate::BitsPerSec(kTargetBitrateBps),
3827 DataRate::BitsPerSec(kTargetBitrateBps),
3828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003829 test::FrameForwarder source;
3830 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003831 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003832
3833 // Insert a single frame, triggering initial configuration.
3834 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3836
3837 EXPECT_EQ(
3838 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3839 kDefaultFramerate);
3840
3841 // Trigger reconfigure encoder (without resetting the entire instance).
3842 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003843 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003844 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3845 video_encoder_config.number_of_streams = 1;
3846 video_encoder_config.video_stream_factory =
3847 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3848 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003849 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3851
3852 // Detector should be updated with fps limit from codec config.
3853 EXPECT_EQ(
3854 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3855 kFramerate);
3856
3857 // Trigger overuse, max framerate should be reduced.
3858 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3859 stats.input_frame_rate = kFramerate;
3860 stats_proxy_->SetMockStats(stats);
3861 video_stream_encoder_->TriggerCpuOveruse();
3862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3863 int adapted_framerate =
3864 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3865 EXPECT_LT(adapted_framerate, kFramerate);
3866
3867 // Trigger underuse, max framerate should go back to codec configured fps.
3868 // Set extra low fps, to make sure it's actually reset, not just incremented.
3869 stats = stats_proxy_->GetStats();
3870 stats.input_frame_rate = adapted_framerate / 2;
3871 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003872 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3874 EXPECT_EQ(
3875 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3876 kFramerate);
3877
3878 video_stream_encoder_->Stop();
3879}
3880
3881TEST_F(VideoStreamEncoderTest,
3882 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3883 const int kFrameWidth = 1280;
3884 const int kFrameHeight = 720;
3885 const int kLowFramerate = 15;
3886 const int kHighFramerate = 25;
3887
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003889 DataRate::BitsPerSec(kTargetBitrateBps),
3890 DataRate::BitsPerSec(kTargetBitrateBps),
3891 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003892 test::FrameForwarder source;
3893 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003894 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003895
3896 // Trigger initial configuration.
3897 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003898 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003899 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3900 video_encoder_config.number_of_streams = 1;
3901 video_encoder_config.video_stream_factory =
3902 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3903 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3904 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003905 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003906 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3907
3908 EXPECT_EQ(
3909 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3910 kLowFramerate);
3911
3912 // Trigger overuse, max framerate should be reduced.
3913 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3914 stats.input_frame_rate = kLowFramerate;
3915 stats_proxy_->SetMockStats(stats);
3916 video_stream_encoder_->TriggerCpuOveruse();
3917 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3918 int adapted_framerate =
3919 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3920 EXPECT_LT(adapted_framerate, kLowFramerate);
3921
3922 // Reconfigure the encoder with a new (higher max framerate), max fps should
3923 // still respect the adaptation.
3924 video_encoder_config.video_stream_factory =
3925 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3926 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3927 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003928 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003929 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3930
3931 EXPECT_EQ(
3932 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3933 adapted_framerate);
3934
3935 // Trigger underuse, max framerate should go back to codec configured fps.
3936 stats = stats_proxy_->GetStats();
3937 stats.input_frame_rate = adapted_framerate;
3938 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003939 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003940 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3941 EXPECT_EQ(
3942 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3943 kHighFramerate);
3944
3945 video_stream_encoder_->Stop();
3946}
3947
mflodmancc3d4422017-08-03 08:27:51 -07003948TEST_F(VideoStreamEncoderTest,
3949 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003950 const int kFrameWidth = 1280;
3951 const int kFrameHeight = 720;
3952 const int kFramerate = 24;
3953
Henrik Boström381d1092020-05-12 18:49:07 +02003954 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003955 DataRate::BitsPerSec(kTargetBitrateBps),
3956 DataRate::BitsPerSec(kTargetBitrateBps),
3957 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003958 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003959 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003960 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003961
3962 // Trigger initial configuration.
3963 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003964 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07003965 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3966 video_encoder_config.number_of_streams = 1;
3967 video_encoder_config.video_stream_factory =
3968 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3969 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07003970 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003971 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07003973
Niels Möller7dc26b72017-12-06 10:27:48 +01003974 EXPECT_EQ(
3975 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3976 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003977
3978 // Trigger overuse, max framerate should be reduced.
3979 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3980 stats.input_frame_rate = kFramerate;
3981 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07003982 video_stream_encoder_->TriggerCpuOveruse();
3983 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01003984 int adapted_framerate =
3985 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07003986 EXPECT_LT(adapted_framerate, kFramerate);
3987
3988 // Change degradation preference to not enable framerate scaling. Target
3989 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02003990 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003991 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01003992 EXPECT_EQ(
3993 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3994 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07003995
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07003997}
3998
mflodmancc3d4422017-08-03 08:27:51 -07003999TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004000 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004002 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4003 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4004 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004005 const int kWidth = 640;
4006 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004007
asaperssonfab67072017-04-04 05:51:49 -07004008 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004009
4010 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004011 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004012
4013 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004014 EXPECT_TRUE_WAIT(
4015 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004016
sprangc5d62e22017-04-02 23:53:04 -07004017 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004018
asaperssonfab67072017-04-04 05:51:49 -07004019 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004020 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004021 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004022
4023 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004024 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004025
Henrik Boström2671dac2020-05-19 16:29:09 +02004026 EXPECT_TRUE_WAIT(
4027 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004028
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004030}
4031
mflodmancc3d4422017-08-03 08:27:51 -07004032TEST_F(VideoStreamEncoderTest,
4033 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004034 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004036 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4037 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4038 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004039 const int kWidth = 640;
4040 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004041
4042 // We expect the n initial frames to get dropped.
4043 int i;
4044 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004045 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004046 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004047 }
4048 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004049 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004050 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004051
4052 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004053 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004054
mflodmancc3d4422017-08-03 08:27:51 -07004055 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004056}
4057
mflodmancc3d4422017-08-03 08:27:51 -07004058TEST_F(VideoStreamEncoderTest,
4059 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004060 const int kWidth = 640;
4061 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004063 DataRate::BitsPerSec(kLowTargetBitrateBps),
4064 DataRate::BitsPerSec(kLowTargetBitrateBps),
4065 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004066
4067 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004068 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004069 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004070
asaperssonfab67072017-04-04 05:51:49 -07004071 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004072 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004073 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004074
mflodmancc3d4422017-08-03 08:27:51 -07004075 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004076}
4077
mflodmancc3d4422017-08-03 08:27:51 -07004078TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004079 const int kWidth = 640;
4080 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004081 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004082
4083 VideoEncoderConfig video_encoder_config;
4084 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4085 // Make format different, to force recreation of encoder.
4086 video_encoder_config.video_format.parameters["foo"] = "foo";
4087 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004088 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004090 DataRate::BitsPerSec(kLowTargetBitrateBps),
4091 DataRate::BitsPerSec(kLowTargetBitrateBps),
4092 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004093
kthelgasonb83797b2017-02-14 11:57:25 -08004094 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004095 video_stream_encoder_->SetSource(&video_source_,
4096 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004097
asaperssonfab67072017-04-04 05:51:49 -07004098 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004099 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004100 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004101
mflodmancc3d4422017-08-03 08:27:51 -07004102 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004103 fake_encoder_.SetQualityScaling(true);
4104}
4105
Åsa Persson139f4dc2019-08-02 09:29:58 +02004106TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4107 webrtc::test::ScopedFieldTrials field_trials(
4108 "WebRTC-Video-QualityScalerSettings/"
4109 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4110 // Reset encoder for field trials to take effect.
4111 ConfigureEncoder(video_encoder_config_.Copy());
4112 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4113 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4114 const int kWidth = 640;
4115 const int kHeight = 360;
4116
Henrik Boström381d1092020-05-12 18:49:07 +02004117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004118 DataRate::BitsPerSec(kTargetBitrateBps),
4119 DataRate::BitsPerSec(kTargetBitrateBps),
4120 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004121 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4122 // Frame should not be dropped.
4123 WaitForEncodedFrame(1);
4124
Henrik Boström381d1092020-05-12 18:49:07 +02004125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004126 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4127 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4128 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004129 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4130 // Frame should not be dropped.
4131 WaitForEncodedFrame(2);
4132
Henrik Boström381d1092020-05-12 18:49:07 +02004133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004134 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4135 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4136 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004137 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4138 // Expect to drop this frame, the wait should time out.
4139 ExpectDroppedFrame();
4140
4141 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004142 EXPECT_TRUE_WAIT(
4143 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004144 video_stream_encoder_->Stop();
4145}
4146
Åsa Perssone644a032019-11-08 15:56:00 +01004147TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4148 webrtc::test::ScopedFieldTrials field_trials(
4149 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4150
4151 // Reset encoder for field trials to take effect.
4152 VideoEncoderConfig config = video_encoder_config_.Copy();
4153 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004154 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004155 ConfigureEncoder(std::move(config));
4156 fake_encoder_.SetQp(kQpLow);
4157
4158 // Enable MAINTAIN_FRAMERATE preference.
4159 AdaptingFrameForwarder source;
4160 source.set_adaptation_enabled(true);
4161 video_stream_encoder_->SetSource(&source,
4162 DegradationPreference::MAINTAIN_FRAMERATE);
4163
4164 // Start at low bitrate.
4165 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004166 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4167 DataRate::BitsPerSec(kLowBitrateBps),
4168 DataRate::BitsPerSec(kLowBitrateBps),
4169 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004170
4171 // Expect first frame to be dropped and resolution to be limited.
4172 const int kWidth = 1280;
4173 const int kHeight = 720;
4174 const int64_t kFrameIntervalMs = 100;
4175 int64_t timestamp_ms = kFrameIntervalMs;
4176 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4177 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004178 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4179 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004180
4181 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4183 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004184
4185 // Insert frames and advance |min_duration_ms|.
4186 for (size_t i = 1; i <= 10; i++) {
4187 timestamp_ms += kFrameIntervalMs;
4188 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4189 WaitForEncodedFrame(timestamp_ms);
4190 }
4191 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4192 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4193
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004194 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004195
4196 // Insert frame should trigger high BW and release quality limitation.
4197 timestamp_ms += kFrameIntervalMs;
4198 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4199 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004200 // The ramp-up code involves the adaptation queue, give it time to execute.
4201 // TODO(hbos): Can we await an appropriate event instead?
4202 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004203 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004204
4205 // Frame should not be adapted.
4206 timestamp_ms += kFrameIntervalMs;
4207 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4208 WaitForEncodedFrame(kWidth, kHeight);
4209 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4210
4211 video_stream_encoder_->Stop();
4212}
4213
mflodmancc3d4422017-08-03 08:27:51 -07004214TEST_F(VideoStreamEncoderTest,
Evan Shrubsoleba8abbb2020-08-13 09:34:09 +02004215 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
4216 AdaptingFrameForwarder source;
4217 source.set_adaptation_enabled(true);
4218 video_stream_encoder_->SetSource(&source,
4219 DegradationPreference::MAINTAIN_FRAMERATE);
4220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4221 DataRate::BitsPerSec(kTargetBitrateBps),
4222 DataRate::BitsPerSec(kTargetBitrateBps),
4223 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4224 fake_encoder_.SetQp(kQpHigh + 1);
4225 const int kWidth = 1280;
4226 const int kHeight = 720;
4227 const int64_t kFrameIntervalMs = 100;
4228 int64_t timestamp_ms = kFrameIntervalMs;
4229 for (size_t i = 1; i <= 100; i++) {
4230 timestamp_ms += kFrameIntervalMs;
4231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4232 WaitForEncodedFrame(timestamp_ms);
4233 }
4234 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
4235 // for the first time.
4236 // TODO(eshr): We should avoid these waits by using threads with simulated
4237 // time.
4238 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
4239 2000 * 2.5 * 2);
4240 timestamp_ms += kFrameIntervalMs;
4241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4242 WaitForEncodedFrame(timestamp_ms);
4243 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4244 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
4245 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
4246 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4247
4248 // Disable Quality scaling by turning off scaler on the encoder and
4249 // reconfiguring.
4250 fake_encoder_.SetQualityScaling(false);
4251 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
4252 kMaxPayloadLength);
4253 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4254 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
4255 // Since we turned off the quality scaler, the adaptations made by it are
4256 // removed.
4257 EXPECT_THAT(source.sink_wants(), ResolutionMax());
4258 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4259
4260 video_stream_encoder_->Stop();
4261}
4262
4263TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004264 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4265 const int kTooSmallWidth = 10;
4266 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004268 DataRate::BitsPerSec(kTargetBitrateBps),
4269 DataRate::BitsPerSec(kTargetBitrateBps),
4270 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004271
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004272 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004273 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004274 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004275 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004276 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004277 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4278
4279 // Trigger adapt down, too small frame, expect no change.
4280 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004281 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004282 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004283 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004284 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4285 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4286
mflodmancc3d4422017-08-03 08:27:51 -07004287 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004288}
4289
mflodmancc3d4422017-08-03 08:27:51 -07004290TEST_F(VideoStreamEncoderTest,
4291 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004292 const int kTooSmallWidth = 10;
4293 const int kTooSmallHeight = 10;
4294 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004295 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004296 DataRate::BitsPerSec(kTargetBitrateBps),
4297 DataRate::BitsPerSec(kTargetBitrateBps),
4298 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004299
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004300 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004301 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004302 video_stream_encoder_->SetSource(&source,
4303 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004304 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004305 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4306 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4307
4308 // Trigger adapt down, expect limited framerate.
4309 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004310 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004311 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004312 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004313 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4314 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4316
4317 // Trigger adapt down, too small frame, expect no change.
4318 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004319 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004320 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004321 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004322 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4323 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
mflodmancc3d4422017-08-03 08:27:51 -07004326 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004327}
4328
mflodmancc3d4422017-08-03 08:27:51 -07004329TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004330 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004332 DataRate::BitsPerSec(kTargetBitrateBps),
4333 DataRate::BitsPerSec(kTargetBitrateBps),
4334 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004335 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004336 const int kFrameWidth = 1280;
4337 const int kFrameHeight = 720;
4338 video_source_.IncomingCapturedFrame(
4339 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004340 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004341 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004342}
4343
sprangb1ca0732017-02-01 08:38:12 -08004344// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004345TEST_F(VideoStreamEncoderTest,
4346 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004348 DataRate::BitsPerSec(kTargetBitrateBps),
4349 DataRate::BitsPerSec(kTargetBitrateBps),
4350 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004351
4352 const int kFrameWidth = 1280;
4353 const int kFrameHeight = 720;
4354 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004355 // requested by
4356 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004357 video_source_.set_adaptation_enabled(true);
4358
4359 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004360 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004361 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004362
4363 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004364 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004365 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004366 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004367 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004368
asaperssonfab67072017-04-04 05:51:49 -07004369 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004370 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004371 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004372 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004373 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004374
mflodmancc3d4422017-08-03 08:27:51 -07004375 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004376}
sprangfe627f32017-03-29 08:24:59 -07004377
mflodmancc3d4422017-08-03 08:27:51 -07004378TEST_F(VideoStreamEncoderTest,
4379 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004380 const int kFrameWidth = 1280;
4381 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004382
Henrik Boström381d1092020-05-12 18:49:07 +02004383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004384 DataRate::BitsPerSec(kTargetBitrateBps),
4385 DataRate::BitsPerSec(kTargetBitrateBps),
4386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004387 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004388 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004389 video_source_.set_adaptation_enabled(true);
4390
sprang4847ae62017-06-27 07:06:52 -07004391 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004392
4393 video_source_.IncomingCapturedFrame(
4394 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004395 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004396
4397 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004398 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004399
4400 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004401 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004402 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004403 video_source_.IncomingCapturedFrame(
4404 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004405 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004406 }
4407
4408 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004409 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004410 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004411 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004412 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004413 video_source_.IncomingCapturedFrame(
4414 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004415 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004416 ++num_frames_dropped;
4417 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004418 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004419 }
4420 }
4421
sprang4847ae62017-06-27 07:06:52 -07004422 // Add some slack to account for frames dropped by the frame dropper.
4423 const int kErrorMargin = 1;
4424 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004425 kErrorMargin);
4426
4427 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004428 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004429 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004430 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004431 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004432 video_source_.IncomingCapturedFrame(
4433 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004434 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004435 ++num_frames_dropped;
4436 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004437 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004438 }
4439 }
sprang4847ae62017-06-27 07:06:52 -07004440 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004441 kErrorMargin);
4442
4443 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004444 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004445 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004446 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004447 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004448 video_source_.IncomingCapturedFrame(
4449 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004450 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004451 ++num_frames_dropped;
4452 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004453 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004454 }
4455 }
sprang4847ae62017-06-27 07:06:52 -07004456 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004457 kErrorMargin);
4458
4459 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004460 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004461 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004462 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004463 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004464 video_source_.IncomingCapturedFrame(
4465 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004466 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004467 ++num_frames_dropped;
4468 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004469 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004470 }
4471 }
4472 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4473
mflodmancc3d4422017-08-03 08:27:51 -07004474 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004475}
4476
mflodmancc3d4422017-08-03 08:27:51 -07004477TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004478 const int kFramerateFps = 5;
4479 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004480 const int kFrameWidth = 1280;
4481 const int kFrameHeight = 720;
4482
sprang4847ae62017-06-27 07:06:52 -07004483 // Reconfigure encoder with two temporal layers and screensharing, which will
4484 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004485 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004486
Henrik Boström381d1092020-05-12 18:49:07 +02004487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004488 DataRate::BitsPerSec(kTargetBitrateBps),
4489 DataRate::BitsPerSec(kTargetBitrateBps),
4490 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004491 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004492 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004493 video_source_.set_adaptation_enabled(true);
4494
sprang4847ae62017-06-27 07:06:52 -07004495 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004496
4497 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004498 rtc::VideoSinkWants last_wants;
4499 do {
4500 last_wants = video_source_.sink_wants();
4501
sprangc5d62e22017-04-02 23:53:04 -07004502 // Insert frames to get a new fps estimate...
4503 for (int j = 0; j < kFramerateFps; ++j) {
4504 video_source_.IncomingCapturedFrame(
4505 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004506 if (video_source_.last_sent_width()) {
4507 sink_.WaitForEncodedFrame(timestamp_ms);
4508 }
sprangc5d62e22017-04-02 23:53:04 -07004509 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004510 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004511 }
4512 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004513 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004514 } while (video_source_.sink_wants().max_framerate_fps <
4515 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004516
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004517 EXPECT_THAT(video_source_.sink_wants(),
4518 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004519
mflodmancc3d4422017-08-03 08:27:51 -07004520 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004521}
asaperssonf7e294d2017-06-13 23:25:22 -07004522
mflodmancc3d4422017-08-03 08:27:51 -07004523TEST_F(VideoStreamEncoderTest,
4524 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004525 const int kWidth = 1280;
4526 const int kHeight = 720;
4527 const int64_t kFrameIntervalMs = 150;
4528 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004530 DataRate::BitsPerSec(kTargetBitrateBps),
4531 DataRate::BitsPerSec(kTargetBitrateBps),
4532 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004533
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004534 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004535 AdaptingFrameForwarder source;
4536 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004537 video_stream_encoder_->SetSource(&source,
4538 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004539 timestamp_ms += kFrameIntervalMs;
4540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004541 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004542 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004543 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4545 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4546
4547 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004548 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004549 timestamp_ms += kFrameIntervalMs;
4550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004552 EXPECT_THAT(source.sink_wants(),
4553 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4556 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4557
4558 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004559 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004560 timestamp_ms += kFrameIntervalMs;
4561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004562 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004563 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4566 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4567
4568 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004569 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004570 timestamp_ms += kFrameIntervalMs;
4571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004572 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004573 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4576 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4577
4578 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004579 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004580 timestamp_ms += kFrameIntervalMs;
4581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004582 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004583 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4586 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4587
4588 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004589 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004590 timestamp_ms += kFrameIntervalMs;
4591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004592 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004593 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4596 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4597
4598 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004599 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004600 timestamp_ms += kFrameIntervalMs;
4601 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004602 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004603 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4606 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4607
4608 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004609 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004610 timestamp_ms += kFrameIntervalMs;
4611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004612 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004613 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004614 rtc::VideoSinkWants last_wants = source.sink_wants();
4615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4617 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4618
4619 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004620 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004621 timestamp_ms += kFrameIntervalMs;
4622 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004623 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004624 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004625 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4627 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4628
Evan Shrubsole64469032020-06-11 10:45:29 +02004629 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004630 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004631 timestamp_ms += kFrameIntervalMs;
4632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004633 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004634 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4637 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4638
4639 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004640 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004641 timestamp_ms += kFrameIntervalMs;
4642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004643 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004644 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4647 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4648
4649 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004650 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004651 timestamp_ms += kFrameIntervalMs;
4652 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004653 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004654 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004655 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4657 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4658
4659 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004660 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004661 timestamp_ms += kFrameIntervalMs;
4662 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004663 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004664 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004665 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4666 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4667 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4668
4669 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004670 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004671 timestamp_ms += kFrameIntervalMs;
4672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004673 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004674 EXPECT_THAT(source.sink_wants(), FpsMax());
4675 EXPECT_EQ(source.sink_wants().max_pixel_count,
4676 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004677 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4679 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4680
4681 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004682 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004683 timestamp_ms += kFrameIntervalMs;
4684 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004685 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004687 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4688 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4689 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4690
Åsa Persson30ab0152019-08-27 12:22:33 +02004691 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004692 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004693 timestamp_ms += kFrameIntervalMs;
4694 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004695 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004696 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004697 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4700 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4701
4702 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004703 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004704 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004705 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4706
mflodmancc3d4422017-08-03 08:27:51 -07004707 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004708}
4709
mflodmancc3d4422017-08-03 08:27:51 -07004710TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004711 const int kWidth = 1280;
4712 const int kHeight = 720;
4713 const int64_t kFrameIntervalMs = 150;
4714 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004716 DataRate::BitsPerSec(kTargetBitrateBps),
4717 DataRate::BitsPerSec(kTargetBitrateBps),
4718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004719
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004720 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004721 AdaptingFrameForwarder source;
4722 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004723 video_stream_encoder_->SetSource(&source,
4724 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004725 timestamp_ms += kFrameIntervalMs;
4726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004727 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4732 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4733 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4735
4736 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004737 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004738 timestamp_ms += kFrameIntervalMs;
4739 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004740 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004741 EXPECT_THAT(source.sink_wants(),
4742 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4744 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4745 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4746 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4747 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4748 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4749
4750 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004751 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004752 timestamp_ms += kFrameIntervalMs;
4753 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004754 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004755 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004756 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4757 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4758 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4760 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4761 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4762
4763 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004764 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004765 timestamp_ms += kFrameIntervalMs;
4766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004767 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004768 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004769 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004770 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4771 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4772 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4773 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4774 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4775
Evan Shrubsole64469032020-06-11 10:45:29 +02004776 // Trigger cpu adapt up, expect no change since QP is most limited.
4777 {
4778 // Store current sink wants since we expect no change and if there is no
4779 // change then last_wants() is not updated.
4780 auto previous_sink_wants = source.sink_wants();
4781 video_stream_encoder_->TriggerCpuUnderuse();
4782 timestamp_ms += kFrameIntervalMs;
4783 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4784 WaitForEncodedFrame(timestamp_ms);
4785 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4786 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4787 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4788 }
4789
4790 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4791 video_stream_encoder_->TriggerQualityHigh();
4792 timestamp_ms += kFrameIntervalMs;
4793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4794 WaitForEncodedFrame(timestamp_ms);
4795 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4796 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4798 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4800 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4801 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4802
4803 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4804 // expect increased resolution (960x540@30fps).
4805 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004806 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004807 timestamp_ms += kFrameIntervalMs;
4808 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004809 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004810 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004811 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4813 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4814 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4815 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004816 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004817
Evan Shrubsole64469032020-06-11 10:45:29 +02004818 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4819 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004820 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004821 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004822 timestamp_ms += kFrameIntervalMs;
4823 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004824 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004825 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004826 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4829 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4830 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4831 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004832 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004833
4834 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004835 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004836 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004837 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004838 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004839
mflodmancc3d4422017-08-03 08:27:51 -07004840 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004841}
4842
mflodmancc3d4422017-08-03 08:27:51 -07004843TEST_F(VideoStreamEncoderTest,
4844 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004845 const int kWidth = 640;
4846 const int kHeight = 360;
4847 const int kFpsLimit = 15;
4848 const int64_t kFrameIntervalMs = 150;
4849 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004851 DataRate::BitsPerSec(kTargetBitrateBps),
4852 DataRate::BitsPerSec(kTargetBitrateBps),
4853 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004854
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004855 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004856 AdaptingFrameForwarder source;
4857 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004858 video_stream_encoder_->SetSource(&source,
4859 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004860 timestamp_ms += kFrameIntervalMs;
4861 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004862 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004863 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4868 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4869 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4870
4871 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004872 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004873 timestamp_ms += kFrameIntervalMs;
4874 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004875 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004876 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004877 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4878 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4879 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4880 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4881 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4882 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4883
4884 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004885 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004886 timestamp_ms += kFrameIntervalMs;
4887 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004888 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004889 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004890 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004891 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07004892 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4893 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4894 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4896
Evan Shrubsole64469032020-06-11 10:45:29 +02004897 // Trigger cpu adapt up, expect no change because quality is most limited.
4898 {
4899 auto previous_sink_wants = source.sink_wants();
4900 // Store current sink wants since we expect no change ind if there is no
4901 // change then last__wants() is not updated.
4902 video_stream_encoder_->TriggerCpuUnderuse();
4903 timestamp_ms += kFrameIntervalMs;
4904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4905 WaitForEncodedFrame(timestamp_ms);
4906 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4907 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4908 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4909 }
4910
4911 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
4912 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004913 timestamp_ms += kFrameIntervalMs;
4914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004915 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004916 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004917 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4918 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4919 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004920 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4921 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4922 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004923
Evan Shrubsole64469032020-06-11 10:45:29 +02004924 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004925 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02004926 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004927 timestamp_ms += kFrameIntervalMs;
4928 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004929 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004930 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004931 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4933 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4934 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4935 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004936 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004937
4938 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004939 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004940 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004941 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004942 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004943
mflodmancc3d4422017-08-03 08:27:51 -07004944 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004945}
4946
mflodmancc3d4422017-08-03 08:27:51 -07004947TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004948 const int kFrameWidth = 1920;
4949 const int kFrameHeight = 1080;
4950 // 3/4 of 1920.
4951 const int kAdaptedFrameWidth = 1440;
4952 // 3/4 of 1080 rounded down to multiple of 4.
4953 const int kAdaptedFrameHeight = 808;
4954 const int kFramerate = 24;
4955
Henrik Boström381d1092020-05-12 18:49:07 +02004956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004957 DataRate::BitsPerSec(kTargetBitrateBps),
4958 DataRate::BitsPerSec(kTargetBitrateBps),
4959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004960 // Trigger reconfigure encoder (without resetting the entire instance).
4961 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004962 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004963 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4964 video_encoder_config.number_of_streams = 1;
4965 video_encoder_config.video_stream_factory =
4966 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004967 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004968 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004969 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004970
4971 video_source_.set_adaptation_enabled(true);
4972
4973 video_source_.IncomingCapturedFrame(
4974 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004975 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004976
4977 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004978 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004979 video_source_.IncomingCapturedFrame(
4980 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004981 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004982
mflodmancc3d4422017-08-03 08:27:51 -07004983 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004984}
4985
mflodmancc3d4422017-08-03 08:27:51 -07004986TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004987 const int kFrameWidth = 1280;
4988 const int kFrameHeight = 720;
4989 const int kLowFps = 2;
4990 const int kHighFps = 30;
4991
Henrik Boström381d1092020-05-12 18:49:07 +02004992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004993 DataRate::BitsPerSec(kTargetBitrateBps),
4994 DataRate::BitsPerSec(kTargetBitrateBps),
4995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004996
4997 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4998 max_framerate_ = kLowFps;
4999
5000 // Insert 2 seconds of 2fps video.
5001 for (int i = 0; i < kLowFps * 2; ++i) {
5002 video_source_.IncomingCapturedFrame(
5003 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5004 WaitForEncodedFrame(timestamp_ms);
5005 timestamp_ms += 1000 / kLowFps;
5006 }
5007
5008 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02005009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005010 DataRate::BitsPerSec(kTargetBitrateBps),
5011 DataRate::BitsPerSec(kTargetBitrateBps),
5012 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005013 video_source_.IncomingCapturedFrame(
5014 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5015 WaitForEncodedFrame(timestamp_ms);
5016 timestamp_ms += 1000 / kLowFps;
5017
5018 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5019
5020 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005021 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005022 const int kFrameIntervalMs = 1000 / kHighFps;
5023 max_framerate_ = kHighFps;
5024 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5025 video_source_.IncomingCapturedFrame(
5026 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5027 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5028 // be dropped if the encoder hans't been updated with the new higher target
5029 // framerate yet, causing it to overshoot the target bitrate and then
5030 // suffering the wrath of the media optimizer.
5031 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5032 timestamp_ms += kFrameIntervalMs;
5033 }
5034
5035 // Don expect correct measurement just yet, but it should be higher than
5036 // before.
5037 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5038
mflodmancc3d4422017-08-03 08:27:51 -07005039 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005040}
5041
mflodmancc3d4422017-08-03 08:27:51 -07005042TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005043 const int kFrameWidth = 1280;
5044 const int kFrameHeight = 720;
5045 const int kTargetBitrateBps = 1000000;
5046
5047 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005048 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005050 DataRate::BitsPerSec(kTargetBitrateBps),
5051 DataRate::BitsPerSec(kTargetBitrateBps),
5052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005053 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005054
5055 // Insert a first video frame, causes another bitrate update.
5056 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5057 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5058 video_source_.IncomingCapturedFrame(
5059 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5060 WaitForEncodedFrame(timestamp_ms);
5061
5062 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5064 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5065 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005066
5067 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005068 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005069 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005070
5071 // Bitrate observer should not be called.
5072 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5073 video_source_.IncomingCapturedFrame(
5074 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5075 ExpectDroppedFrame();
5076
mflodmancc3d4422017-08-03 08:27:51 -07005077 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005078}
ilnik6b826ef2017-06-16 06:53:48 -07005079
Niels Möller4db138e2018-04-19 09:04:13 +02005080TEST_F(VideoStreamEncoderTest,
5081 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5082 const int kFrameWidth = 1280;
5083 const int kFrameHeight = 720;
5084 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005086 DataRate::BitsPerSec(kTargetBitrateBps),
5087 DataRate::BitsPerSec(kTargetBitrateBps),
5088 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005089 video_source_.IncomingCapturedFrame(
5090 CreateFrame(1, kFrameWidth, kFrameHeight));
5091 WaitForEncodedFrame(1);
5092 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5093 .low_encode_usage_threshold_percent,
5094 default_options.low_encode_usage_threshold_percent);
5095 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5096 .high_encode_usage_threshold_percent,
5097 default_options.high_encode_usage_threshold_percent);
5098 video_stream_encoder_->Stop();
5099}
5100
5101TEST_F(VideoStreamEncoderTest,
5102 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5103 const int kFrameWidth = 1280;
5104 const int kFrameHeight = 720;
5105 CpuOveruseOptions hardware_options;
5106 hardware_options.low_encode_usage_threshold_percent = 150;
5107 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005108 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005109
Henrik Boström381d1092020-05-12 18:49:07 +02005110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005111 DataRate::BitsPerSec(kTargetBitrateBps),
5112 DataRate::BitsPerSec(kTargetBitrateBps),
5113 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005114 video_source_.IncomingCapturedFrame(
5115 CreateFrame(1, kFrameWidth, kFrameHeight));
5116 WaitForEncodedFrame(1);
5117 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5118 .low_encode_usage_threshold_percent,
5119 hardware_options.low_encode_usage_threshold_percent);
5120 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5121 .high_encode_usage_threshold_percent,
5122 hardware_options.high_encode_usage_threshold_percent);
5123 video_stream_encoder_->Stop();
5124}
5125
Niels Möller6bb5ab92019-01-11 11:11:10 +01005126TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5127 const int kFrameWidth = 320;
5128 const int kFrameHeight = 240;
5129 const int kFps = 30;
5130 const int kTargetBitrateBps = 120000;
5131 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5132
Henrik Boström381d1092020-05-12 18:49:07 +02005133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005134 DataRate::BitsPerSec(kTargetBitrateBps),
5135 DataRate::BitsPerSec(kTargetBitrateBps),
5136 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005137
5138 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5139 max_framerate_ = kFps;
5140
5141 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5142 fake_encoder_.SimulateOvershoot(1.0);
5143 int num_dropped = 0;
5144 for (int i = 0; i < kNumFramesInRun; ++i) {
5145 video_source_.IncomingCapturedFrame(
5146 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5147 // Wait up to two frame durations for a frame to arrive.
5148 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5149 ++num_dropped;
5150 }
5151 timestamp_ms += 1000 / kFps;
5152 }
5153
Erik Språnga8d48ab2019-02-08 14:17:40 +01005154 // Framerate should be measured to be near the expected target rate.
5155 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5156
5157 // Frame drops should be within 5% of expected 0%.
5158 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005159
5160 // Make encoder produce frames at double the expected bitrate during 3 seconds
5161 // of video, verify number of drops. Rate needs to be slightly changed in
5162 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005163 double overshoot_factor = 2.0;
5164 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5165 // With bitrate adjuster, when need to overshoot even more to trigger
5166 // frame dropping.
5167 overshoot_factor *= 2;
5168 }
5169 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005171 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5172 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5173 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005174 num_dropped = 0;
5175 for (int i = 0; i < kNumFramesInRun; ++i) {
5176 video_source_.IncomingCapturedFrame(
5177 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5178 // Wait up to two frame durations for a frame to arrive.
5179 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5180 ++num_dropped;
5181 }
5182 timestamp_ms += 1000 / kFps;
5183 }
5184
Henrik Boström381d1092020-05-12 18:49:07 +02005185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005186 DataRate::BitsPerSec(kTargetBitrateBps),
5187 DataRate::BitsPerSec(kTargetBitrateBps),
5188 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005189
5190 // Target framerate should be still be near the expected target, despite
5191 // the frame drops.
5192 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5193
5194 // Frame drops should be within 5% of expected 50%.
5195 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005196
5197 video_stream_encoder_->Stop();
5198}
5199
5200TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5201 const int kFrameWidth = 320;
5202 const int kFrameHeight = 240;
5203 const int kActualInputFps = 24;
5204 const int kTargetBitrateBps = 120000;
5205
5206 ASSERT_GT(max_framerate_, kActualInputFps);
5207
5208 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5209 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005211 DataRate::BitsPerSec(kTargetBitrateBps),
5212 DataRate::BitsPerSec(kTargetBitrateBps),
5213 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005214
5215 // Insert 3 seconds of video, with an input fps lower than configured max.
5216 for (int i = 0; i < kActualInputFps * 3; ++i) {
5217 video_source_.IncomingCapturedFrame(
5218 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5219 // Wait up to two frame durations for a frame to arrive.
5220 WaitForEncodedFrame(timestamp_ms);
5221 timestamp_ms += 1000 / kActualInputFps;
5222 }
5223
5224 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5225
5226 video_stream_encoder_->Stop();
5227}
5228
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005229TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5230 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005232 DataRate::BitsPerSec(kTargetBitrateBps),
5233 DataRate::BitsPerSec(kTargetBitrateBps),
5234 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005235
5236 fake_encoder_.BlockNextEncode();
5237 video_source_.IncomingCapturedFrame(
5238 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5239 WaitForEncodedFrame(1);
5240 // On the very first frame full update should be forced.
5241 rect = fake_encoder_.GetLastUpdateRect();
5242 EXPECT_EQ(rect.offset_x, 0);
5243 EXPECT_EQ(rect.offset_y, 0);
5244 EXPECT_EQ(rect.height, codec_height_);
5245 EXPECT_EQ(rect.width, codec_width_);
5246 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5247 // call to ContinueEncode.
5248 video_source_.IncomingCapturedFrame(
5249 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5250 ExpectDroppedFrame();
5251 video_source_.IncomingCapturedFrame(
5252 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5253 ExpectDroppedFrame();
5254 fake_encoder_.ContinueEncode();
5255 WaitForEncodedFrame(3);
5256 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5257 rect = fake_encoder_.GetLastUpdateRect();
5258 EXPECT_EQ(rect.offset_x, 1);
5259 EXPECT_EQ(rect.offset_y, 0);
5260 EXPECT_EQ(rect.width, 10);
5261 EXPECT_EQ(rect.height, 1);
5262
5263 video_source_.IncomingCapturedFrame(
5264 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5265 WaitForEncodedFrame(4);
5266 // Previous frame was encoded, so no accumulation should happen.
5267 rect = fake_encoder_.GetLastUpdateRect();
5268 EXPECT_EQ(rect.offset_x, 0);
5269 EXPECT_EQ(rect.offset_y, 0);
5270 EXPECT_EQ(rect.width, 1);
5271 EXPECT_EQ(rect.height, 1);
5272
5273 video_stream_encoder_->Stop();
5274}
5275
Erik Språngd7329ca2019-02-21 21:19:53 +01005276TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005278 DataRate::BitsPerSec(kTargetBitrateBps),
5279 DataRate::BitsPerSec(kTargetBitrateBps),
5280 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005281
5282 // First frame is always keyframe.
5283 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5284 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005285 EXPECT_THAT(
5286 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005287 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005288
5289 // Insert delta frame.
5290 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5291 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005292 EXPECT_THAT(
5293 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005294 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005295
5296 // Request next frame be a key-frame.
5297 video_stream_encoder_->SendKeyFrame();
5298 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5299 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005300 EXPECT_THAT(
5301 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005302 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005303
5304 video_stream_encoder_->Stop();
5305}
5306
5307TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5308 // Setup simulcast with three streams.
5309 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005311 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5312 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5313 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005314 // Wait for all three layers before triggering event.
5315 sink_.SetNumExpectedLayers(3);
5316
5317 // First frame is always keyframe.
5318 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5319 WaitForEncodedFrame(1);
5320 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005321 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5322 VideoFrameType::kVideoFrameKey,
5323 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005324
5325 // Insert delta frame.
5326 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5327 WaitForEncodedFrame(2);
5328 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005329 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5330 VideoFrameType::kVideoFrameDelta,
5331 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005332
5333 // Request next frame be a key-frame.
5334 // Only first stream is configured to produce key-frame.
5335 video_stream_encoder_->SendKeyFrame();
5336 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5337 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005338
5339 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5340 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005341 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005342 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005343 VideoFrameType::kVideoFrameKey,
5344 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005345
5346 video_stream_encoder_->Stop();
5347}
5348
5349TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5350 // Configure internal source factory and setup test again.
5351 encoder_factory_.SetHasInternalSource(true);
5352 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005354 DataRate::BitsPerSec(kTargetBitrateBps),
5355 DataRate::BitsPerSec(kTargetBitrateBps),
5356 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005357
5358 // Call encoder directly, simulating internal source where encoded frame
5359 // callback in VideoStreamEncoder is called despite no OnFrame().
5360 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5361 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005362 EXPECT_THAT(
5363 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005364 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005365
Niels Möller8f7ce222019-03-21 15:43:58 +01005366 const std::vector<VideoFrameType> kDeltaFrame = {
5367 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005368 // Need to set timestamp manually since manually for injected frame.
5369 VideoFrame frame = CreateFrame(101, nullptr);
5370 frame.set_timestamp(101);
5371 fake_encoder_.InjectFrame(frame, false);
5372 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005373 EXPECT_THAT(
5374 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005375 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005376
5377 // Request key-frame. The forces a dummy frame down into the encoder.
5378 fake_encoder_.ExpectNullFrame();
5379 video_stream_encoder_->SendKeyFrame();
5380 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005381 EXPECT_THAT(
5382 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005383 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005384
5385 video_stream_encoder_->Stop();
5386}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005387
5388TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5389 // Configure internal source factory and setup test again.
5390 encoder_factory_.SetHasInternalSource(true);
5391 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005393 DataRate::BitsPerSec(kTargetBitrateBps),
5394 DataRate::BitsPerSec(kTargetBitrateBps),
5395 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005396
5397 int64_t timestamp = 1;
5398 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005399 image.SetEncodedData(
5400 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005401 image.capture_time_ms_ = ++timestamp;
5402 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5403 const int64_t kEncodeFinishDelayMs = 10;
5404 image.timing_.encode_start_ms = timestamp;
5405 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5406 fake_encoder_.InjectEncodedImage(image);
5407 // Wait for frame without incrementing clock.
5408 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5409 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5410 // capture timestamp should be kEncodeFinishDelayMs in the past.
5411 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5412 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5413 kEncodeFinishDelayMs);
5414
5415 video_stream_encoder_->Stop();
5416}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005417
5418TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005419 // SPS contains VUI with restrictions on the maximum number of reordered
5420 // pictures, there is no need to rewrite the bitstream to enable faster
5421 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005422 ResetEncoder("H264", 1, 1, 1, false);
5423
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5425 DataRate::BitsPerSec(kTargetBitrateBps),
5426 DataRate::BitsPerSec(kTargetBitrateBps),
5427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5428 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005429
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005430 fake_encoder_.SetEncodedImageData(
5431 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005432
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005433 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5434 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005435
5436 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5437 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005438
5439 video_stream_encoder_->Stop();
5440}
5441
5442TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005443 // SPS does not contain VUI, the bitstream is will be rewritten with added
5444 // VUI with restrictions on the maximum number of reordered pictures to
5445 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005446 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5447 0x00, 0x00, 0x03, 0x03, 0xF4,
5448 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005449 ResetEncoder("H264", 1, 1, 1, false);
5450
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5452 DataRate::BitsPerSec(kTargetBitrateBps),
5453 DataRate::BitsPerSec(kTargetBitrateBps),
5454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5455 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005456
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005457 fake_encoder_.SetEncodedImageData(
5458 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005459
Mirta Dvornicic97910da2020-07-14 15:29:23 +02005460 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5461 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005462
5463 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5464 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005465
5466 video_stream_encoder_->Stop();
5467}
5468
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005469TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5470 const int kFrameWidth = 1280;
5471 const int kFrameHeight = 720;
5472 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5473
Henrik Boström381d1092020-05-12 18:49:07 +02005474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005475 DataRate::BitsPerSec(kTargetBitrateBps),
5476 DataRate::BitsPerSec(kTargetBitrateBps),
5477 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005478 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5479
5480 // Insert a first video frame. It should be dropped because of downscale in
5481 // resolution.
5482 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5483 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5484 frame.set_rotation(kVideoRotation_270);
5485 video_source_.IncomingCapturedFrame(frame);
5486
5487 ExpectDroppedFrame();
5488
5489 // Second frame is downscaled.
5490 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5491 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5492 frame.set_rotation(kVideoRotation_90);
5493 video_source_.IncomingCapturedFrame(frame);
5494
5495 WaitForEncodedFrame(timestamp_ms);
5496 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5497
5498 // Insert another frame, also downscaled.
5499 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5500 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5501 frame.set_rotation(kVideoRotation_180);
5502 video_source_.IncomingCapturedFrame(frame);
5503
5504 WaitForEncodedFrame(timestamp_ms);
5505 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5506
5507 video_stream_encoder_->Stop();
5508}
5509
Erik Språng5056af02019-09-02 15:53:11 +02005510TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5511 const int kFrameWidth = 320;
5512 const int kFrameHeight = 180;
5513
5514 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005516 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5517 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5518 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005519 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005520 /*rtt_ms=*/0,
5521 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005522
5523 // Insert a first video frame so that encoder gets configured.
5524 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5525 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5526 frame.set_rotation(kVideoRotation_270);
5527 video_source_.IncomingCapturedFrame(frame);
5528 WaitForEncodedFrame(timestamp_ms);
5529
5530 // Set a target rate below the minimum allowed by the codec settings.
5531 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005532 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5533 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005535 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005536 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005537 /*link_allocation=*/target_rate,
5538 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005539 /*rtt_ms=*/0,
5540 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005541 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5542
5543 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5544 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5545 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005546 DataRate allocation_sum =
5547 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005548 EXPECT_EQ(min_rate, allocation_sum);
5549 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5550
5551 video_stream_encoder_->Stop();
5552}
5553
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005554TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005555 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005556 DataRate::BitsPerSec(kTargetBitrateBps),
5557 DataRate::BitsPerSec(kTargetBitrateBps),
5558 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005559 // Capture a frame and wait for it to synchronize with the encoder thread.
5560 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5561 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5562 WaitForEncodedFrame(1);
5563
5564 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5565 ASSERT_TRUE(prev_rate_settings.has_value());
5566 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5567 kDefaultFramerate);
5568
5569 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5570 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5571 timestamp_ms += 1000 / kDefaultFramerate;
5572 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5573 WaitForEncodedFrame(timestamp_ms);
5574 }
5575 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5576 kDefaultFramerate);
5577 // Capture larger frame to trigger a reconfigure.
5578 codec_height_ *= 2;
5579 codec_width_ *= 2;
5580 timestamp_ms += 1000 / kDefaultFramerate;
5581 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5582 WaitForEncodedFrame(timestamp_ms);
5583
5584 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5585 auto current_rate_settings =
5586 fake_encoder_.GetAndResetLastRateControlSettings();
5587 // Ensure we have actually reconfigured twice
5588 // The rate settings should have been set again even though
5589 // they haven't changed.
5590 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005591 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005592
5593 video_stream_encoder_->Stop();
5594}
5595
philipeld9cc8c02019-09-16 14:53:40 +02005596struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005597 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5598 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5599 MOCK_METHOD(void,
5600 RequestEncoderSwitch,
5601 (const webrtc::SdpVideoFormat& format),
5602 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005603};
5604
5605TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5606 constexpr int kDontCare = 100;
5607
5608 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5609 video_send_config_.encoder_settings.encoder_switch_request_callback =
5610 &switch_callback;
5611 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5612 encoder_config.codec_type = kVideoCodecVP8;
5613 webrtc::test::ScopedFieldTrials field_trial(
5614 "WebRTC-NetworkCondition-EncoderSwitch/"
5615 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5616 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5617
5618 // Reset encoder for new configuration to take effect.
5619 ConfigureEncoder(std::move(encoder_config));
5620
5621 // Send one frame to trigger ReconfigureEncoder.
5622 video_source_.IncomingCapturedFrame(
5623 CreateFrame(kDontCare, kDontCare, kDontCare));
5624
5625 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005626 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5627 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005628 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005629 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005630
Henrik Boström381d1092020-05-12 18:49:07 +02005631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005632 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5633 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5634 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005635 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005636 /*rtt_ms=*/0,
5637 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005638
5639 video_stream_encoder_->Stop();
5640}
5641
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005642TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5643 constexpr int kDontCare = 100;
5644
5645 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5646 video_send_config_.encoder_settings.encoder_switch_request_callback =
5647 &switch_callback;
5648 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5649 encoder_config.codec_type = kVideoCodecVP8;
5650 webrtc::test::ScopedFieldTrials field_trial(
5651 "WebRTC-NetworkCondition-EncoderSwitch/"
5652 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5653 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5654
5655 // Reset encoder for new configuration to take effect.
5656 ConfigureEncoder(std::move(encoder_config));
5657
5658 // Send one frame to trigger ReconfigureEncoder.
5659 video_source_.IncomingCapturedFrame(
5660 CreateFrame(kDontCare, kDontCare, kDontCare));
5661
5662 using Config = EncoderSwitchRequestCallback::Config;
5663 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5664 .Times(0);
5665
Henrik Boström381d1092020-05-12 18:49:07 +02005666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005667 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5668 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5669 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5670 /*fraction_lost=*/0,
5671 /*rtt_ms=*/0,
5672 /*cwnd_reduce_ratio=*/0);
5673
5674 video_stream_encoder_->Stop();
5675}
5676
philipeld9cc8c02019-09-16 14:53:40 +02005677TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5678 constexpr int kSufficientBitrateToNotDrop = 1000;
5679 constexpr int kHighRes = 500;
5680 constexpr int kLowRes = 100;
5681
5682 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5683 video_send_config_.encoder_settings.encoder_switch_request_callback =
5684 &switch_callback;
5685 webrtc::test::ScopedFieldTrials field_trial(
5686 "WebRTC-NetworkCondition-EncoderSwitch/"
5687 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5688 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5689 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5690 encoder_config.codec_type = kVideoCodecH264;
5691
5692 // Reset encoder for new configuration to take effect.
5693 ConfigureEncoder(std::move(encoder_config));
5694
5695 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5696 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5697 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005699 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5700 /*stable_target_bitrate=*/
5701 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5702 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005703 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005704 /*rtt_ms=*/0,
5705 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005706
5707 // Send one frame to trigger ReconfigureEncoder.
5708 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5709 WaitForEncodedFrame(1);
5710
5711 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005712 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5713 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005714 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005715 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005716
5717 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5718 WaitForEncodedFrame(2);
5719
5720 video_stream_encoder_->Stop();
5721}
5722
philipel9b058032020-02-10 11:30:00 +01005723TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5724 constexpr int kDontCare = 100;
5725 StrictMock<MockEncoderSelector> encoder_selector;
5726 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5727 &fake_encoder_, &encoder_selector);
5728 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5729
5730 // Reset encoder for new configuration to take effect.
5731 ConfigureEncoder(video_encoder_config_.Copy());
5732
5733 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5734
5735 video_source_.IncomingCapturedFrame(
5736 CreateFrame(kDontCare, kDontCare, kDontCare));
5737 video_stream_encoder_->Stop();
5738
5739 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5740 // to it's factory, so in order for the encoder instance in the
5741 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5742 // reset the |video_stream_encoder_| here.
5743 video_stream_encoder_.reset();
5744}
5745
5746TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5747 constexpr int kDontCare = 100;
5748
5749 NiceMock<MockEncoderSelector> encoder_selector;
5750 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5751 video_send_config_.encoder_settings.encoder_switch_request_callback =
5752 &switch_callback;
5753 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5754 &fake_encoder_, &encoder_selector);
5755 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5756
5757 // Reset encoder for new configuration to take effect.
5758 ConfigureEncoder(video_encoder_config_.Copy());
5759
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005760 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005761 .WillByDefault(Return(SdpVideoFormat("AV1")));
5762 EXPECT_CALL(switch_callback,
5763 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5764 Field(&SdpVideoFormat::name, "AV1"))));
5765
Henrik Boström381d1092020-05-12 18:49:07 +02005766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005767 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5768 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5769 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005770 /*fraction_lost=*/0,
5771 /*rtt_ms=*/0,
5772 /*cwnd_reduce_ratio=*/0);
5773
5774 video_stream_encoder_->Stop();
5775}
5776
5777TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5778 constexpr int kSufficientBitrateToNotDrop = 1000;
5779 constexpr int kDontCare = 100;
5780
5781 NiceMock<MockVideoEncoder> video_encoder;
5782 NiceMock<MockEncoderSelector> encoder_selector;
5783 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5784 video_send_config_.encoder_settings.encoder_switch_request_callback =
5785 &switch_callback;
5786 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5787 &video_encoder, &encoder_selector);
5788 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5789
5790 // Reset encoder for new configuration to take effect.
5791 ConfigureEncoder(video_encoder_config_.Copy());
5792
5793 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5794 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5795 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005797 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5798 /*stable_target_bitrate=*/
5799 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5800 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005801 /*fraction_lost=*/0,
5802 /*rtt_ms=*/0,
5803 /*cwnd_reduce_ratio=*/0);
5804
5805 ON_CALL(video_encoder, Encode(_, _))
5806 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5807 ON_CALL(encoder_selector, OnEncoderBroken())
5808 .WillByDefault(Return(SdpVideoFormat("AV2")));
5809
5810 rtc::Event encode_attempted;
5811 EXPECT_CALL(switch_callback,
5812 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5813 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5814 EXPECT_EQ(format.name, "AV2");
5815 encode_attempted.Set();
5816 });
5817
5818 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5819 encode_attempted.Wait(3000);
5820
5821 video_stream_encoder_->Stop();
5822
5823 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5824 // to it's factory, so in order for the encoder instance in the
5825 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5826 // reset the |video_stream_encoder_| here.
5827 video_stream_encoder_.reset();
5828}
5829
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005830TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005831 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005832 const int kFrameWidth = 320;
5833 const int kFrameHeight = 180;
5834
5835 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005836 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005838 /*target_bitrate=*/rate,
5839 /*stable_target_bitrate=*/rate,
5840 /*link_allocation=*/rate,
5841 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005842 /*rtt_ms=*/0,
5843 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005844
5845 // Insert a first video frame so that encoder gets configured.
5846 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5847 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5848 frame.set_rotation(kVideoRotation_270);
5849 video_source_.IncomingCapturedFrame(frame);
5850 WaitForEncodedFrame(timestamp_ms);
5851 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5852
5853 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005854 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005856 /*target_bitrate=*/new_stable_rate,
5857 /*stable_target_bitrate=*/new_stable_rate,
5858 /*link_allocation=*/rate,
5859 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005860 /*rtt_ms=*/0,
5861 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5863 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5864 video_stream_encoder_->Stop();
5865}
5866
5867TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005868 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005869 const int kFrameWidth = 320;
5870 const int kFrameHeight = 180;
5871
5872 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005873 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005875 /*target_bitrate=*/rate,
5876 /*stable_target_bitrate=*/rate,
5877 /*link_allocation=*/rate,
5878 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005879 /*rtt_ms=*/0,
5880 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005881
5882 // Insert a first video frame so that encoder gets configured.
5883 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5884 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5885 frame.set_rotation(kVideoRotation_270);
5886 video_source_.IncomingCapturedFrame(frame);
5887 WaitForEncodedFrame(timestamp_ms);
5888 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5889
5890 // Set a higher target rate without changing the link_allocation. Should not
5891 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005892 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005894 /*target_bitrate=*/rate,
5895 /*stable_target_bitrate=*/new_stable_rate,
5896 /*link_allocation=*/rate,
5897 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005898 /*rtt_ms=*/0,
5899 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005900 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5901 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5902 video_stream_encoder_->Stop();
5903}
5904
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005905TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5906 test::ScopedFieldTrials field_trials(
5907 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5908 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5909 const int kFramerateFps = 30;
5910 const int kWidth = 1920;
5911 const int kHeight = 1080;
5912 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5913 // Works on screenshare mode.
5914 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5915 // We rely on the automatic resolution adaptation, but we handle framerate
5916 // adaptation manually by mocking the stats proxy.
5917 video_source_.set_adaptation_enabled(true);
5918
5919 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005921 DataRate::BitsPerSec(kTargetBitrateBps),
5922 DataRate::BitsPerSec(kTargetBitrateBps),
5923 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005924 video_stream_encoder_->SetSource(&video_source_,
5925 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005926 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005927
5928 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5929 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5930
5931 // Pass enough frames with the full update to trigger animation detection.
5932 for (int i = 0; i < kNumFrames; ++i) {
5933 int64_t timestamp_ms =
5934 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5935 frame.set_ntp_time_ms(timestamp_ms);
5936 frame.set_timestamp_us(timestamp_ms * 1000);
5937 video_source_.IncomingCapturedFrame(frame);
5938 WaitForEncodedFrame(timestamp_ms);
5939 }
5940
5941 // Resolution should be limited.
5942 rtc::VideoSinkWants expected;
5943 expected.max_framerate_fps = kFramerateFps;
5944 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005945 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005946
5947 // Pass one frame with no known update.
5948 // Resolution cap should be removed immediately.
5949 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5950 frame.set_ntp_time_ms(timestamp_ms);
5951 frame.set_timestamp_us(timestamp_ms * 1000);
5952 frame.clear_update_rect();
5953
5954 video_source_.IncomingCapturedFrame(frame);
5955 WaitForEncodedFrame(timestamp_ms);
5956
5957 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005958 EXPECT_THAT(video_source_.sink_wants(),
5959 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005960
5961 video_stream_encoder_->Stop();
5962}
5963
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02005964TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
5965 const int kWidth = 720; // 540p adapted down.
5966 const int kHeight = 405;
5967 const int kNumFrames = 3;
5968 // Works on screenshare mode.
5969 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5970 /*num_spatial_layers=*/2, /*screenshare=*/true);
5971
5972 video_source_.set_adaptation_enabled(true);
5973
5974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5975 DataRate::BitsPerSec(kTargetBitrateBps),
5976 DataRate::BitsPerSec(kTargetBitrateBps),
5977 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5978
5979 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5980
5981 // Pass enough frames with the full update to trigger animation detection.
5982 for (int i = 0; i < kNumFrames; ++i) {
5983 int64_t timestamp_ms =
5984 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5985 frame.set_ntp_time_ms(timestamp_ms);
5986 frame.set_timestamp_us(timestamp_ms * 1000);
5987 video_source_.IncomingCapturedFrame(frame);
5988 WaitForEncodedFrame(timestamp_ms);
5989 }
5990
5991 video_stream_encoder_->Stop();
5992}
5993
perkj26091b12016-09-01 01:17:40 -07005994} // namespace webrtc