blob: 6a11bf6b83b769a21fb2f4a6579e6c91849b7fb3 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020024#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010025#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020026#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020027#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010028#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020029#include "call/adaptation/test/fake_adaptation_constraint.h"
30#include "call/adaptation/test/fake_adaptation_listener.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Sergey Silkin86684962018-03-28 19:32:37 +020035#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020036#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010037#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "rtc_base/fake_clock.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020039#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020040#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020042#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010043#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020044#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020045#include "system_wrappers/include/sleep.h"
46#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020052#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020053#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070054
55namespace webrtc {
56
sprang57c2fff2017-01-16 06:24:02 -080057using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020058using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020059using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020060using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Ge;
62using ::testing::Gt;
63using ::testing::Le;
64using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010065using ::testing::Matcher;
66using ::testing::NiceMock;
67using ::testing::Return;
philipeld9cc8c02019-09-16 14:53:40 +020068using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080069
perkj803d97f2016-11-01 11:45:46 -070070namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020071const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010072const int kQpLow = 1;
73const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinFramerateFps = 2;
75const int kMinBalancedFramerateFps = 7;
76const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080077const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010078const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020079const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010080const uint32_t kSimulcastTargetBitrateBps = 3150000;
81const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080082const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070083const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020084const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020085const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020086const VideoEncoder::ResolutionBitrateLimits
87 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
88const VideoEncoder::ResolutionBitrateLimits
89 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080090
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020091uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
92 0x00, 0x00, 0x03, 0x03, 0xF4,
93 0x05, 0x03, 0xC7, 0xE0, 0x1B,
94 0x41, 0x10, 0x8D, 0x00};
95
perkj803d97f2016-11-01 11:45:46 -070096class TestBuffer : public webrtc::I420Buffer {
97 public:
98 TestBuffer(rtc::Event* event, int width, int height)
99 : I420Buffer(width, height), event_(event) {}
100
101 private:
102 friend class rtc::RefCountedObject<TestBuffer>;
103 ~TestBuffer() override {
104 if (event_)
105 event_->Set();
106 }
107 rtc::Event* const event_;
108};
109
Noah Richards51db4212019-06-12 06:59:12 -0700110// A fake native buffer that can't be converted to I420.
111class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
112 public:
113 FakeNativeBuffer(rtc::Event* event, int width, int height)
114 : event_(event), width_(width), height_(height) {}
115 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
116 int width() const override { return width_; }
117 int height() const override { return height_; }
118 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
119 return nullptr;
120 }
121
122 private:
123 friend class rtc::RefCountedObject<FakeNativeBuffer>;
124 ~FakeNativeBuffer() override {
125 if (event_)
126 event_->Set();
127 }
128 rtc::Event* const event_;
129 const int width_;
130 const int height_;
131};
132
Niels Möller7dc26b72017-12-06 10:27:48 +0100133class CpuOveruseDetectorProxy : public OveruseFrameDetector {
134 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200135 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
136 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200137 last_target_framerate_fps_(-1),
138 framerate_updated_event_(true /* manual_reset */,
139 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100140 virtual ~CpuOveruseDetectorProxy() {}
141
142 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200143 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100144 last_target_framerate_fps_ = framerate_fps;
145 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200146 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100147 }
148
149 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200150 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100151 return last_target_framerate_fps_;
152 }
153
Niels Möller4db138e2018-04-19 09:04:13 +0200154 CpuOveruseOptions GetOptions() { return options_; }
155
Henrik Boström381d1092020-05-12 18:49:07 +0200156 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
157
Niels Möller7dc26b72017-12-06 10:27:48 +0100158 private:
Markus Handella3765182020-07-08 13:13:32 +0200159 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100160 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200161 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100162};
163
Henrik Boström91aa7322020-04-28 12:24:33 +0200164class FakeQualityScalerQpUsageHandlerCallback
165 : public QualityScalerQpUsageHandlerCallbackInterface {
166 public:
167 FakeQualityScalerQpUsageHandlerCallback()
Henrik Boström5bf60e42020-05-13 15:20:25 +0200168 : QualityScalerQpUsageHandlerCallbackInterface(),
169 qp_usage_handled_event_(/*manual_reset=*/true,
170 /*initially_signaled=*/false),
171 clear_qp_samples_result_(absl::nullopt) {}
172 ~FakeQualityScalerQpUsageHandlerCallback() override {
173 RTC_DCHECK(clear_qp_samples_result_.has_value());
174 }
Henrik Boström91aa7322020-04-28 12:24:33 +0200175
176 void OnQpUsageHandled(bool clear_qp_samples) override {
177 clear_qp_samples_result_ = clear_qp_samples;
Henrik Boström5bf60e42020-05-13 15:20:25 +0200178 qp_usage_handled_event_.Set();
Henrik Boström91aa7322020-04-28 12:24:33 +0200179 }
180
Henrik Boström5bf60e42020-05-13 15:20:25 +0200181 bool WaitForQpUsageHandled() { return qp_usage_handled_event_.Wait(5000); }
182
Henrik Boström91aa7322020-04-28 12:24:33 +0200183 absl::optional<bool> clear_qp_samples_result() const {
184 return clear_qp_samples_result_;
185 }
186
187 private:
Henrik Boström5bf60e42020-05-13 15:20:25 +0200188 rtc::Event qp_usage_handled_event_;
Henrik Boström91aa7322020-04-28 12:24:33 +0200189 absl::optional<bool> clear_qp_samples_result_;
190};
191
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200192class FakeVideoSourceRestrictionsListener
193 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200194 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200195 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200196 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200197 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200198 RTC_DCHECK(was_restrictions_updated_);
199 }
200
201 rtc::Event* restrictions_updated_event() {
202 return &restrictions_updated_event_;
203 }
204
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200205 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200206 void OnVideoSourceRestrictionsUpdated(
207 VideoSourceRestrictions restrictions,
208 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200209 rtc::scoped_refptr<Resource> reason,
210 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200211 was_restrictions_updated_ = true;
212 restrictions_updated_event_.Set();
213 }
214
215 private:
216 bool was_restrictions_updated_;
217 rtc::Event restrictions_updated_event_;
218};
219
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200220auto WantsFps(Matcher<int> fps_matcher) {
221 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
222 fps_matcher);
223}
224
225auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
226 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
227 AllOf(max_pixel_matcher, Gt(0)));
228}
229
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200230auto ResolutionMax() {
231 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200232 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200233 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
234 Eq(absl::nullopt)));
235}
236
237auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239}
240
241auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200242 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200243}
244
245auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200246 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200247}
248
249auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200250 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200251}
252
253auto FpsMaxResolutionMax() {
254 return AllOf(FpsMax(), ResolutionMax());
255}
256
257auto UnlimitedSinkWants() {
258 return AllOf(FpsUnlimited(), ResolutionMax());
259}
260
261auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
262 Matcher<int> fps_range_matcher;
263
264 if (last_frame_pixels <= 320 * 240) {
265 fps_range_matcher = AllOf(Ge(7), Le(10));
266 } else if (last_frame_pixels <= 480 * 270) {
267 fps_range_matcher = AllOf(Ge(10), Le(15));
268 } else if (last_frame_pixels <= 640 * 480) {
269 fps_range_matcher = Ge(15);
270 } else {
271 fps_range_matcher = Eq(kDefaultFramerate);
272 }
273 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
274 fps_range_matcher);
275}
276
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200277auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
278 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
279 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
280}
281
282auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
283 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
284}
285
286auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
287 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
288}
289
290auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
291 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
292 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
293}
294
295auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
296 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
297 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
298}
299
300auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
301 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
302 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
303}
304
305auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
306 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
307 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
308}
309
mflodmancc3d4422017-08-03 08:27:51 -0700310class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700311 public:
Niels Möller213618e2018-07-24 09:29:58 +0200312 VideoStreamEncoderUnderTest(SendStatisticsProxy* stats_proxy,
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200313 const VideoStreamEncoderSettings& settings,
314 TaskQueueFactory* task_queue_factory)
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100315 : VideoStreamEncoder(Clock::GetRealTimeClock(),
316 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200317 stats_proxy,
318 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200319 std::unique_ptr<OveruseFrameDetector>(
320 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100321 new CpuOveruseDetectorProxy(stats_proxy)),
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100322 task_queue_factory),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200323 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200324 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200325 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200326 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200327 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200328 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200329 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100330 }
perkj803d97f2016-11-01 11:45:46 -0700331
Henrik Boström381d1092020-05-12 18:49:07 +0200332 void SetSourceAndWaitForRestrictionsUpdated(
333 rtc::VideoSourceInterface<VideoFrame>* source,
334 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200335 FakeVideoSourceRestrictionsListener listener;
336 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200337 SetSource(source, degradation_preference);
338 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200339 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200340 }
341
342 void SetSourceAndWaitForFramerateUpdated(
343 rtc::VideoSourceInterface<VideoFrame>* source,
344 const DegradationPreference& degradation_preference) {
345 overuse_detector_proxy_->framerate_updated_event()->Reset();
346 SetSource(source, degradation_preference);
347 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
348 }
349
350 void OnBitrateUpdatedAndWaitForManagedResources(
351 DataRate target_bitrate,
352 DataRate stable_target_bitrate,
353 DataRate link_allocation,
354 uint8_t fraction_lost,
355 int64_t round_trip_time_ms,
356 double cwnd_reduce_ratio) {
357 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
358 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
359 // Bitrate is updated on the encoder queue.
360 WaitUntilTaskQueueIsIdle();
361 // Give the managed resources time to react to the new bitrate.
362 // TODO(hbos): Can we await an appropriate event instead?
363 WaitUntilAdaptationTaskQueueIsIdle();
364 }
365
366 void WaitUntilAdaptationTaskQueueIsIdle() {
367 rtc::Event event;
368 resource_adaptation_queue()->PostTask([&event] { event.Set(); });
369 ASSERT_TRUE(event.Wait(5000));
370 }
371
kthelgason2fc52542017-03-03 00:24:41 -0800372 // This is used as a synchronisation mechanism, to make sure that the
373 // encoder queue is not blocked before we start sending it frames.
374 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100375 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200376 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800377 ASSERT_TRUE(event.Wait(5000));
378 }
379
Henrik Boström91aa7322020-04-28 12:24:33 +0200380 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200381 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200383 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200384 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200385 event.Set();
386 });
387 ASSERT_TRUE(event.Wait(5000));
388 }
389 void TriggerCpuUnderuse() {
390 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200391 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200392 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 event.Set();
394 });
395 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200396 }
kthelgason876222f2016-11-29 01:44:11 -0800397
Henrik Boström91aa7322020-04-28 12:24:33 +0200398 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200399 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200400 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200401 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200402 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 event.Set();
404 });
405 ASSERT_TRUE(event.Wait(5000));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200406 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200407 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 rtc::Event event;
Henrik Boström381d1092020-05-12 18:49:07 +0200409 resource_adaptation_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200410 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 event.Set();
412 });
413 ASSERT_TRUE(event.Wait(5000));
414 }
415
416 // Fakes high QP resource usage measurements on the real
417 // QualityScalerResource. Returns whether or not QP samples would have been
418 // cleared if this had been a real signal from the QualityScaler.
419 bool TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 rtc::scoped_refptr<FakeQualityScalerQpUsageHandlerCallback> callback =
421 new FakeQualityScalerQpUsageHandlerCallback();
Henrik Boström5bf60e42020-05-13 15:20:25 +0200422 encoder_queue()->PostTask([this, callback] {
423 // This will cause a "ping" between adaptation task queue and encoder
424 // queue. When we have the result, the |callback| will be notified.
Henrik Boström91aa7322020-04-28 12:24:33 +0200425 quality_scaler_resource_for_testing()->OnReportQpUsageHigh(callback);
Henrik Boström91aa7322020-04-28 12:24:33 +0200426 });
Henrik Boström5bf60e42020-05-13 15:20:25 +0200427 EXPECT_TRUE(callback->WaitForQpUsageHandled());
Henrik Boström91aa7322020-04-28 12:24:33 +0200428 EXPECT_TRUE(callback->clear_qp_samples_result().has_value());
429 return callback->clear_qp_samples_result().value();
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200430 }
sprangfda496a2017-06-15 04:21:07 -0700431
Niels Möller7dc26b72017-12-06 10:27:48 +0100432 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200433 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
434 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200435 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700436};
437
asapersson5f7226f2016-11-25 04:37:00 -0800438class VideoStreamFactory
439 : public VideoEncoderConfig::VideoStreamFactoryInterface {
440 public:
sprangfda496a2017-06-15 04:21:07 -0700441 explicit VideoStreamFactory(size_t num_temporal_layers, int framerate)
442 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
asapersson5f7226f2016-11-25 04:37:00 -0800443 EXPECT_GT(num_temporal_layers, 0u);
sprangfda496a2017-06-15 04:21:07 -0700444 EXPECT_GT(framerate, 0);
asapersson5f7226f2016-11-25 04:37:00 -0800445 }
446
447 private:
448 std::vector<VideoStream> CreateEncoderStreams(
449 int width,
450 int height,
451 const VideoEncoderConfig& encoder_config) override {
452 std::vector<VideoStream> streams =
453 test::CreateVideoStreams(width, height, encoder_config);
454 for (VideoStream& stream : streams) {
Sergey Silkina796a7e2018-03-01 15:11:29 +0100455 stream.num_temporal_layers = num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700456 stream.max_framerate = framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800457 }
458 return streams;
459 }
sprangfda496a2017-06-15 04:21:07 -0700460
asapersson5f7226f2016-11-25 04:37:00 -0800461 const size_t num_temporal_layers_;
sprangfda496a2017-06-15 04:21:07 -0700462 const int framerate_;
asapersson5f7226f2016-11-25 04:37:00 -0800463};
464
Noah Richards51db4212019-06-12 06:59:12 -0700465// Simulates simulcast behavior and makes highest stream resolutions divisible
466// by 4.
467class CroppingVideoStreamFactory
468 : public VideoEncoderConfig::VideoStreamFactoryInterface {
469 public:
470 explicit CroppingVideoStreamFactory(size_t num_temporal_layers, int framerate)
471 : num_temporal_layers_(num_temporal_layers), framerate_(framerate) {
472 EXPECT_GT(num_temporal_layers, 0u);
473 EXPECT_GT(framerate, 0);
474 }
475
476 private:
477 std::vector<VideoStream> CreateEncoderStreams(
478 int width,
479 int height,
480 const VideoEncoderConfig& encoder_config) override {
481 std::vector<VideoStream> streams = test::CreateVideoStreams(
482 width - width % 4, height - height % 4, encoder_config);
483 for (VideoStream& stream : streams) {
484 stream.num_temporal_layers = num_temporal_layers_;
485 stream.max_framerate = framerate_;
486 }
487 return streams;
488 }
489
490 const size_t num_temporal_layers_;
491 const int framerate_;
492};
493
sprangb1ca0732017-02-01 08:38:12 -0800494class AdaptingFrameForwarder : public test::FrameForwarder {
495 public:
496 AdaptingFrameForwarder() : adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700497 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800498
499 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200500 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800501 adaptation_enabled_ = enabled;
502 }
503
asaperssonfab67072017-04-04 05:51:49 -0700504 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200505 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800506 return adaptation_enabled_;
507 }
508
asapersson09f05612017-05-15 23:40:18 -0700509 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200510 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700511 return last_wants_;
512 }
513
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200514 absl::optional<int> last_sent_width() const { return last_width_; }
515 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800516
sprangb1ca0732017-02-01 08:38:12 -0800517 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
518 int cropped_width = 0;
519 int cropped_height = 0;
520 int out_width = 0;
521 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700522 if (adaption_enabled()) {
523 if (adapter_.AdaptFrameResolution(
524 video_frame.width(), video_frame.height(),
525 video_frame.timestamp_us() * 1000, &cropped_width,
526 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100527 VideoFrame adapted_frame =
528 VideoFrame::Builder()
529 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
530 nullptr, out_width, out_height))
531 .set_timestamp_rtp(99)
532 .set_timestamp_ms(99)
533 .set_rotation(kVideoRotation_0)
534 .build();
sprangc5d62e22017-04-02 23:53:04 -0700535 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100536 if (video_frame.has_update_rect()) {
537 adapted_frame.set_update_rect(
538 video_frame.update_rect().ScaleWithFrame(
539 video_frame.width(), video_frame.height(), 0, 0,
540 video_frame.width(), video_frame.height(), out_width,
541 out_height));
542 }
sprangc5d62e22017-04-02 23:53:04 -0700543 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(adapted_frame.width());
545 last_height_.emplace(adapted_frame.height());
546 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200547 last_width_ = absl::nullopt;
548 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700549 }
sprangb1ca0732017-02-01 08:38:12 -0800550 } else {
551 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800552 last_width_.emplace(video_frame.width());
553 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800554 }
555 }
556
557 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
558 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200559 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200560 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100561 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200562 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800563 }
sprangb1ca0732017-02-01 08:38:12 -0800564 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200565 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
566 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200567 absl::optional<int> last_width_;
568 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800569};
sprangc5d62e22017-04-02 23:53:04 -0700570
Niels Möller213618e2018-07-24 09:29:58 +0200571// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700572class MockableSendStatisticsProxy : public SendStatisticsProxy {
573 public:
574 MockableSendStatisticsProxy(Clock* clock,
575 const VideoSendStream::Config& config,
576 VideoEncoderConfig::ContentType content_type)
577 : SendStatisticsProxy(clock, config, content_type) {}
578
579 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200580 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700581 if (mock_stats_)
582 return *mock_stats_;
583 return SendStatisticsProxy::GetStats();
584 }
585
Niels Möller213618e2018-07-24 09:29:58 +0200586 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200587 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200588 if (mock_stats_)
589 return mock_stats_->input_frame_rate;
590 return SendStatisticsProxy::GetInputFrameRate();
591 }
sprangc5d62e22017-04-02 23:53:04 -0700592 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200593 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700594 mock_stats_.emplace(stats);
595 }
596
597 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700599 mock_stats_.reset();
600 }
601
602 private:
Markus Handella3765182020-07-08 13:13:32 +0200603 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200604 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
sprangc5d62e22017-04-02 23:53:04 -0700605};
606
sprang4847ae62017-06-27 07:06:52 -0700607class MockBitrateObserver : public VideoBitrateAllocationObserver {
608 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200609 MOCK_METHOD(void,
610 OnBitrateAllocationUpdated,
611 (const VideoBitrateAllocation&),
612 (override));
sprang4847ae62017-06-27 07:06:52 -0700613};
614
philipel9b058032020-02-10 11:30:00 +0100615class MockEncoderSelector
616 : public VideoEncoderFactory::EncoderSelectorInterface {
617 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200618 MOCK_METHOD(void,
619 OnCurrentEncoder,
620 (const SdpVideoFormat& format),
621 (override));
622 MOCK_METHOD(absl::optional<SdpVideoFormat>,
623 OnAvailableBitrate,
624 (const DataRate& rate),
625 (override));
626 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100627};
628
perkj803d97f2016-11-01 11:45:46 -0700629} // namespace
630
mflodmancc3d4422017-08-03 08:27:51 -0700631class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700632 public:
633 static const int kDefaultTimeoutMs = 30 * 1000;
634
mflodmancc3d4422017-08-03 08:27:51 -0700635 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700636 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700637 codec_width_(320),
638 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200639 max_framerate_(kDefaultFramerate),
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200640 task_queue_factory_(CreateDefaultTaskQueueFactory()),
perkj26091b12016-09-01 01:17:40 -0700641 fake_encoder_(),
Niels Möller4db138e2018-04-19 09:04:13 +0200642 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700643 stats_proxy_(new MockableSendStatisticsProxy(
perkj803d97f2016-11-01 11:45:46 -0700644 Clock::GetRealTimeClock(),
645 video_send_config_,
646 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
perkj26091b12016-09-01 01:17:40 -0700647 sink_(&fake_encoder_) {}
648
649 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700650 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700651 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200652 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800653 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200654 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200655 video_send_config_.rtp.payload_name = "FAKE";
656 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700657
Per512ecb32016-09-23 15:52:06 +0200658 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200659 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
sprang4847ae62017-06-27 07:06:52 -0700660 video_encoder_config.video_stream_factory =
661 new rtc::RefCountedObject<VideoStreamFactory>(1, max_framerate_);
Erik Språng08127a92016-11-16 16:41:30 +0100662 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700663
664 // Framerate limit is specified by the VideoStreamFactory.
665 std::vector<VideoStream> streams =
666 video_encoder_config.video_stream_factory->CreateEncoderStreams(
667 codec_width_, codec_height_, video_encoder_config);
668 max_framerate_ = streams[0].max_framerate;
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100669 fake_clock_.SetTime(Timestamp::Micros(1234));
sprang4847ae62017-06-27 07:06:52 -0700670
Niels Möllerf1338562018-04-26 09:51:47 +0200671 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800672 }
673
Niels Möllerf1338562018-04-26 09:51:47 +0200674 void ConfigureEncoder(VideoEncoderConfig video_encoder_config) {
mflodmancc3d4422017-08-03 08:27:51 -0700675 if (video_stream_encoder_)
676 video_stream_encoder_->Stop();
677 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Danil Chapovalovd3ba2362019-04-10 17:01:23 +0200678 stats_proxy_.get(), video_send_config_.encoder_settings,
679 task_queue_factory_.get()));
mflodmancc3d4422017-08-03 08:27:51 -0700680 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
681 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700682 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700683 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
684 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200685 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700686 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800687 }
688
689 void ResetEncoder(const std::string& payload_name,
690 size_t num_streams,
691 size_t num_temporal_layers,
emircanbbcc3562017-08-18 00:28:40 -0700692 unsigned char num_spatial_layers,
sprang4847ae62017-06-27 07:06:52 -0700693 bool screenshare) {
Niels Möller259a4972018-04-05 15:36:51 +0200694 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800695
696 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200697 video_encoder_config.codec_type = PayloadStringToCodecType(payload_name);
asapersson5f7226f2016-11-25 04:37:00 -0800698 video_encoder_config.number_of_streams = num_streams;
Erik Språngd7329ca2019-02-21 21:19:53 +0100699 video_encoder_config.max_bitrate_bps =
700 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
asapersson5f7226f2016-11-25 04:37:00 -0800701 video_encoder_config.video_stream_factory =
sprangfda496a2017-06-15 04:21:07 -0700702 new rtc::RefCountedObject<VideoStreamFactory>(num_temporal_layers,
703 kDefaultFramerate);
sprang4847ae62017-06-27 07:06:52 -0700704 video_encoder_config.content_type =
705 screenshare ? VideoEncoderConfig::ContentType::kScreen
706 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700707 if (payload_name == "VP9") {
708 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
709 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
710 video_encoder_config.encoder_specific_settings =
711 new rtc::RefCountedObject<
712 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
713 }
Niels Möllerf1338562018-04-26 09:51:47 +0200714 ConfigureEncoder(std::move(video_encoder_config));
perkj26091b12016-09-01 01:17:40 -0700715 }
716
sprang57c2fff2017-01-16 06:24:02 -0800717 VideoFrame CreateFrame(int64_t ntp_time_ms,
718 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100719 VideoFrame frame =
720 VideoFrame::Builder()
721 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
722 destruction_event, codec_width_, codec_height_))
723 .set_timestamp_rtp(99)
724 .set_timestamp_ms(99)
725 .set_rotation(kVideoRotation_0)
726 .build();
sprang57c2fff2017-01-16 06:24:02 -0800727 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700728 return frame;
729 }
730
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100731 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
732 rtc::Event* destruction_event,
733 int offset_x) const {
734 VideoFrame frame =
735 VideoFrame::Builder()
736 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
737 destruction_event, codec_width_, codec_height_))
738 .set_timestamp_rtp(99)
739 .set_timestamp_ms(99)
740 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100741 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100742 .build();
743 frame.set_ntp_time_ms(ntp_time_ms);
744 return frame;
745 }
746
sprang57c2fff2017-01-16 06:24:02 -0800747 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100748 VideoFrame frame =
749 VideoFrame::Builder()
750 .set_video_frame_buffer(
751 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
752 .set_timestamp_rtp(99)
753 .set_timestamp_ms(99)
754 .set_rotation(kVideoRotation_0)
755 .build();
sprang57c2fff2017-01-16 06:24:02 -0800756 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700757 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700758 return frame;
759 }
760
Noah Richards51db4212019-06-12 06:59:12 -0700761 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
762 rtc::Event* destruction_event,
763 int width,
764 int height) const {
765 VideoFrame frame =
766 VideoFrame::Builder()
767 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
768 destruction_event, width, height))
769 .set_timestamp_rtp(99)
770 .set_timestamp_ms(99)
771 .set_rotation(kVideoRotation_0)
772 .build();
773 frame.set_ntp_time_ms(ntp_time_ms);
774 return frame;
775 }
776
777 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
778 rtc::Event* destruction_event) const {
779 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
780 codec_height_);
781 }
782
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100783 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
784 MockBitrateObserver bitrate_observer;
785 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
786
787 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
788 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +0200789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100790 DataRate::BitsPerSec(kTargetBitrateBps),
791 DataRate::BitsPerSec(kTargetBitrateBps),
792 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100793
794 video_source_.IncomingCapturedFrame(
795 CreateFrame(1, codec_width_, codec_height_));
796 WaitForEncodedFrame(1);
797 }
798
sprang4847ae62017-06-27 07:06:52 -0700799 void WaitForEncodedFrame(int64_t expected_ntp_time) {
800 sink_.WaitForEncodedFrame(expected_ntp_time);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100801 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700802 }
803
804 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
805 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100806 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700807 return ok;
808 }
809
810 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
811 sink_.WaitForEncodedFrame(expected_width, expected_height);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100812 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700813 }
814
815 void ExpectDroppedFrame() {
816 sink_.ExpectDroppedFrame();
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100817 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700818 }
819
820 bool WaitForFrame(int64_t timeout_ms) {
821 bool ok = sink_.WaitForFrame(timeout_ms);
Danil Chapovalov0c626af2020-02-10 11:16:00 +0100822 fake_clock_.AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700823 return ok;
824 }
825
perkj26091b12016-09-01 01:17:40 -0700826 class TestEncoder : public test::FakeEncoder {
827 public:
Niels Möllerc572ff32018-11-07 08:43:50 +0100828 TestEncoder() : FakeEncoder(Clock::GetRealTimeClock()) {}
perkj26091b12016-09-01 01:17:40 -0700829
asaperssonfab67072017-04-04 05:51:49 -0700830 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200831 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700832 return config_;
833 }
834
835 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200836 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700837 block_next_encode_ = true;
838 }
839
Erik Språngaed30702018-11-05 12:57:17 +0100840 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200841 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100842 EncoderInfo info;
Erik Språngb7cb7b52019-02-26 15:52:33 +0100843 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100845 info.scaling_settings = VideoEncoder::ScalingSettings(
846 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100847 }
848 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 for (int i = 0; i < kMaxSpatialLayers; ++i) {
850 if (temporal_layers_supported_[i]) {
851 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
852 info.fps_allocation[i].resize(num_layers);
853 }
854 }
Erik Språngaed30702018-11-05 12:57:17 +0100855 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200856
857 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100858 info.requested_resolution_alignment = requested_resolution_alignment_;
Erik Språngaed30702018-11-05 12:57:17 +0100859 return info;
kthelgason876222f2016-11-29 01:44:11 -0800860 }
861
Erik Språngb7cb7b52019-02-26 15:52:33 +0100862 int32_t RegisterEncodeCompleteCallback(
863 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200864 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100865 encoded_image_callback_ = callback;
866 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
867 }
868
perkjfa10b552016-10-02 23:45:26 -0700869 void ContinueEncode() { continue_encode_event_.Set(); }
870
871 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
872 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200873 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700874 EXPECT_EQ(timestamp_, timestamp);
875 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
876 }
877
kthelgason2fc52542017-03-03 00:24:41 -0800878 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200879 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800880 quality_scaling_ = b;
881 }
kthelgasonad9010c2017-02-14 00:46:51 -0800882
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100883 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200884 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100885 requested_resolution_alignment_ = requested_resolution_alignment;
886 }
887
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100888 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200889 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100890 is_hardware_accelerated_ = is_hardware_accelerated;
891 }
892
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100893 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
894 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200895 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100896 temporal_layers_supported_[spatial_idx] = supported;
897 }
898
Sergey Silkin6456e352019-07-08 17:56:40 +0200899 void SetResolutionBitrateLimits(
900 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200901 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200902 resolution_bitrate_limits_ = thresholds;
903 }
904
sprangfe627f32017-03-29 08:24:59 -0700905 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200906 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700907 force_init_encode_failed_ = force_failure;
908 }
909
Niels Möller6bb5ab92019-01-11 11:11:10 +0100910 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100912 rate_factor_ = rate_factor;
913 }
914
Erik Språngd7329ca2019-02-21 21:19:53 +0100915 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100917 return last_framerate_;
918 }
919
Erik Språngd7329ca2019-02-21 21:19:53 +0100920 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100922 return last_update_rect_;
923 }
924
Niels Möller87e2d782019-03-07 10:18:23 +0100925 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100927 return last_frame_types_;
928 }
929
930 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100931 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100932 keyframe ? VideoFrameType::kVideoFrameKey
933 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100934 {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 last_frame_types_ = frame_type;
937 }
Niels Möllerb859b322019-03-07 12:40:01 +0100938 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100939 }
940
Erik Språngb7cb7b52019-02-26 15:52:33 +0100941 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200942 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100943 encoded_image_callback_->OnEncodedImage(image, nullptr, nullptr);
944 }
945
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200946 void InjectEncodedImage(const EncodedImage& image,
947 const CodecSpecificInfo* codec_specific_info,
948 const RTPFragmentationHeader* fragmentation) {
Markus Handella3765182020-07-08 13:13:32 +0200949 MutexLock lock(&local_mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200950 encoded_image_callback_->OnEncodedImage(image, codec_specific_info,
951 fragmentation);
952 }
953
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200955 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100956 expect_null_frame_ = true;
957 }
958
Erik Språng5056af02019-09-02 15:53:11 +0200959 absl::optional<VideoEncoder::RateControlParameters>
960 GetAndResetLastRateControlSettings() {
961 auto settings = last_rate_control_settings_;
962 last_rate_control_settings_.reset();
963 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 }
965
Sergey Silkin5ee69672019-07-02 14:18:34 +0200966 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200967 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200968 return num_encoder_initializations_;
969 }
970
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200971 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200972 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200973 return num_set_rates_;
974 }
975
perkjfa10b552016-10-02 23:45:26 -0700976 private:
perkj26091b12016-09-01 01:17:40 -0700977 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +0100978 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -0700979 bool block_encode;
980 {
Markus Handella3765182020-07-08 13:13:32 +0200981 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100982 if (expect_null_frame_) {
983 EXPECT_EQ(input_image.timestamp(), 0u);
984 EXPECT_EQ(input_image.width(), 1);
985 last_frame_types_ = *frame_types;
986 expect_null_frame_ = false;
987 } else {
988 EXPECT_GT(input_image.timestamp(), timestamp_);
989 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
990 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
991 }
perkj26091b12016-09-01 01:17:40 -0700992
993 timestamp_ = input_image.timestamp();
994 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -0700995 last_input_width_ = input_image.width();
996 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -0700997 block_encode = block_next_encode_;
998 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100999 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001000 last_frame_types_ = *frame_types;
perkj26091b12016-09-01 01:17:40 -07001001 }
Niels Möllerb859b322019-03-07 12:40:01 +01001002 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001003 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001004 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001005 return result;
1006 }
1007
sprangfe627f32017-03-29 08:24:59 -07001008 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001009 const Settings& settings) override {
1010 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001011
Markus Handella3765182020-07-08 13:13:32 +02001012 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001013 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001014
1015 ++num_encoder_initializations_;
1016
Erik Språng82fad3d2018-03-21 09:57:23 +01001017 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001018 // Simulate setting up temporal layers, in order to validate the life
1019 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001020 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001021 frame_buffer_controller_ =
1022 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001023 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001024 if (force_init_encode_failed_) {
1025 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001026 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001027 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001028
Erik Språngb7cb7b52019-02-26 15:52:33 +01001029 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001030 return res;
1031 }
1032
Erik Språngb7cb7b52019-02-26 15:52:33 +01001033 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001034 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001035 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1036 initialized_ = EncoderState::kUninitialized;
1037 return FakeEncoder::Release();
1038 }
1039
Erik Språng16cb8f52019-04-12 13:59:09 +02001040 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001041 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001042 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001043 VideoBitrateAllocation adjusted_rate_allocation;
1044 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1045 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001046 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001047 adjusted_rate_allocation.SetBitrate(
1048 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001049 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001050 rate_factor_));
1051 }
1052 }
1053 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001054 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001055 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001056 RateControlParameters adjusted_paramters = parameters;
1057 adjusted_paramters.bitrate = adjusted_rate_allocation;
1058 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001059 }
1060
Markus Handella3765182020-07-08 13:13:32 +02001061 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001062 enum class EncoderState {
1063 kUninitialized,
1064 kInitializationFailed,
1065 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001066 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1067 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001068 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001069 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1070 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1071 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1072 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1073 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1074 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
1075 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Elad Aloncde8ab22019-03-20 11:56:20 +01001076 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001077 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001078 absl::optional<bool>
1079 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001080 local_mutex_);
1081 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1082 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1083 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001084 absl::optional<VideoEncoder::RateControlParameters>
1085 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001086 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1087 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001088 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001089 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001090 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1091 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001092 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001093 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001094 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001095 RTC_GUARDED_BY(local_mutex_);
1096 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001097 };
1098
mflodmancc3d4422017-08-03 08:27:51 -07001099 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001100 public:
1101 explicit TestSink(TestEncoder* test_encoder)
Niels Möllerc572ff32018-11-07 08:43:50 +01001102 : test_encoder_(test_encoder) {}
perkj26091b12016-09-01 01:17:40 -07001103
perkj26091b12016-09-01 01:17:40 -07001104 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001105 EXPECT_TRUE(
1106 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1107 }
1108
1109 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1110 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001111 uint32_t timestamp = 0;
sprang4847ae62017-06-27 07:06:52 -07001112 if (!encoded_frame_event_.Wait(timeout_ms))
1113 return false;
perkj26091b12016-09-01 01:17:40 -07001114 {
Markus Handella3765182020-07-08 13:13:32 +02001115 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001116 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001117 }
1118 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001119 return true;
perkj26091b12016-09-01 01:17:40 -07001120 }
1121
sprangb1ca0732017-02-01 08:38:12 -08001122 void WaitForEncodedFrame(uint32_t expected_width,
1123 uint32_t expected_height) {
sprangc5d62e22017-04-02 23:53:04 -07001124 EXPECT_TRUE(encoded_frame_event_.Wait(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001125 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001126 }
1127
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001128 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001129 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001130 uint32_t width = 0;
1131 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001132 {
Markus Handella3765182020-07-08 13:13:32 +02001133 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001134 width = last_width_;
1135 height = last_height_;
1136 }
1137 EXPECT_EQ(expected_height, height);
1138 EXPECT_EQ(expected_width, width);
1139 }
1140
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001141 void CheckLastFrameSizeIsMultipleOf(int resolution_alignment) {
1142 int width = 0;
1143 int height = 0;
1144 {
Markus Handella3765182020-07-08 13:13:32 +02001145 MutexLock lock(&mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001146 width = last_width_;
1147 height = last_height_;
1148 }
1149 EXPECT_EQ(width % resolution_alignment, 0);
1150 EXPECT_EQ(height % resolution_alignment, 0);
1151 }
1152
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001153 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1154 VideoRotation rotation;
1155 {
Markus Handella3765182020-07-08 13:13:32 +02001156 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001157 rotation = last_rotation_;
1158 }
1159 EXPECT_EQ(expected_rotation, rotation);
1160 }
1161
kthelgason2fc52542017-03-03 00:24:41 -08001162 void ExpectDroppedFrame() { EXPECT_FALSE(encoded_frame_event_.Wait(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001163
sprangc5d62e22017-04-02 23:53:04 -07001164 bool WaitForFrame(int64_t timeout_ms) {
1165 return encoded_frame_event_.Wait(timeout_ms);
1166 }
1167
perkj26091b12016-09-01 01:17:40 -07001168 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001169 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001170 expect_frames_ = false;
1171 }
1172
asaperssonfab67072017-04-04 05:51:49 -07001173 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001174 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001175 return number_of_reconfigurations_;
1176 }
1177
asaperssonfab67072017-04-04 05:51:49 -07001178 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001179 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001180 return min_transmit_bitrate_bps_;
1181 }
1182
Erik Språngd7329ca2019-02-21 21:19:53 +01001183 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001184 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001185 num_expected_layers_ = num_layers;
1186 }
1187
Erik Språngb7cb7b52019-02-26 15:52:33 +01001188 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001189 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001190 return last_capture_time_ms_;
1191 }
1192
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001193 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001194 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001195 return std::move(last_encoded_image_data_);
1196 }
1197
1198 RTPFragmentationHeader GetLastFragmentation() {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001200 return std::move(last_fragmentation_);
1201 }
1202
perkj26091b12016-09-01 01:17:40 -07001203 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001204 Result OnEncodedImage(
1205 const EncodedImage& encoded_image,
1206 const CodecSpecificInfo* codec_specific_info,
1207 const RTPFragmentationHeader* fragmentation) override {
Markus Handella3765182020-07-08 13:13:32 +02001208 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001209 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001210 last_encoded_image_data_ = std::vector<uint8_t>(
1211 encoded_image.data(), encoded_image.data() + encoded_image.size());
1212 if (fragmentation) {
1213 last_fragmentation_.CopyFrom(*fragmentation);
1214 }
Erik Språngd7329ca2019-02-21 21:19:53 +01001215 uint32_t timestamp = encoded_image.Timestamp();
1216 if (last_timestamp_ != timestamp) {
1217 num_received_layers_ = 1;
1218 } else {
1219 ++num_received_layers_;
1220 }
1221 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001222 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001223 last_width_ = encoded_image._encodedWidth;
1224 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001225 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001226 if (num_received_layers_ == num_expected_layers_) {
1227 encoded_frame_event_.Set();
1228 }
sprangb1ca0732017-02-01 08:38:12 -08001229 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001230 }
1231
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001232 void OnEncoderConfigurationChanged(
1233 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001234 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001235 VideoEncoderConfig::ContentType content_type,
1236 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001237 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001238 ++number_of_reconfigurations_;
1239 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1240 }
1241
Markus Handella3765182020-07-08 13:13:32 +02001242 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001243 TestEncoder* test_encoder_;
1244 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001245 std::vector<uint8_t> last_encoded_image_data_;
1246 RTPFragmentationHeader last_fragmentation_;
sprangb1ca0732017-02-01 08:38:12 -08001247 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001248 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001249 uint32_t last_height_ = 0;
1250 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001251 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001252 size_t num_expected_layers_ = 1;
1253 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001254 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001255 int number_of_reconfigurations_ = 0;
1256 int min_transmit_bitrate_bps_ = 0;
perkj26091b12016-09-01 01:17:40 -07001257 };
1258
Sergey Silkin5ee69672019-07-02 14:18:34 +02001259 class VideoBitrateAllocatorProxyFactory
1260 : public VideoBitrateAllocatorFactory {
1261 public:
1262 VideoBitrateAllocatorProxyFactory()
1263 : bitrate_allocator_factory_(
1264 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1265
1266 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1267 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001269 codec_config_ = codec;
1270 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1271 }
1272
1273 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001274 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001275 return codec_config_;
1276 }
1277
1278 private:
1279 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1280
Markus Handella3765182020-07-08 13:13:32 +02001281 mutable Mutex mutex_;
1282 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001283 };
1284
perkj26091b12016-09-01 01:17:40 -07001285 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001286 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001287 int codec_width_;
1288 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001289 int max_framerate_;
Erik Språng82268752019-08-29 15:07:47 +02001290 rtc::ScopedFakeClock fake_clock_;
Danil Chapovalovd3ba2362019-04-10 17:01:23 +02001291 const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
perkj26091b12016-09-01 01:17:40 -07001292 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001293 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001294 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001295 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001296 TestSink sink_;
sprangb1ca0732017-02-01 08:38:12 -08001297 AdaptingFrameForwarder video_source_;
mflodmancc3d4422017-08-03 08:27:51 -07001298 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001299};
1300
mflodmancc3d4422017-08-03 08:27:51 -07001301TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001303 DataRate::BitsPerSec(kTargetBitrateBps),
1304 DataRate::BitsPerSec(kTargetBitrateBps),
1305 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001306 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001307 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001308 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001309 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001310 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001311}
1312
mflodmancc3d4422017-08-03 08:27:51 -07001313TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001314 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001315 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001316 // The encoder will cache up to one frame for a short duration. Adding two
1317 // frames means that the first frame will be dropped and the second frame will
1318 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001319 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Sebastian Janssona3177052018-04-10 13:05:49 +02001320 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001321 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001322
Henrik Boström381d1092020-05-12 18:49:07 +02001323 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001324 DataRate::BitsPerSec(kTargetBitrateBps),
1325 DataRate::BitsPerSec(kTargetBitrateBps),
1326 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001327
Sebastian Janssona3177052018-04-10 13:05:49 +02001328 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001329 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001330 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1331
1332 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001333 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001334}
1335
mflodmancc3d4422017-08-03 08:27:51 -07001336TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001337 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001338 DataRate::BitsPerSec(kTargetBitrateBps),
1339 DataRate::BitsPerSec(kTargetBitrateBps),
1340 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001341 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001342 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001343
Henrik Boström381d1092020-05-12 18:49:07 +02001344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1345 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1346 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001347 // The encoder will cache up to one frame for a short duration. Adding two
1348 // frames means that the first frame will be dropped and the second frame will
1349 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001350 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001351 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001352
Henrik Boström381d1092020-05-12 18:49:07 +02001353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001354 DataRate::BitsPerSec(kTargetBitrateBps),
1355 DataRate::BitsPerSec(kTargetBitrateBps),
1356 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001357 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001358 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1359 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001360 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001361}
1362
mflodmancc3d4422017-08-03 08:27:51 -07001363TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001365 DataRate::BitsPerSec(kTargetBitrateBps),
1366 DataRate::BitsPerSec(kTargetBitrateBps),
1367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001368 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001369 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001370
1371 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001372 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001373
perkja49cbd32016-09-16 07:53:41 -07001374 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001375 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001376 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001377}
1378
mflodmancc3d4422017-08-03 08:27:51 -07001379TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001381 DataRate::BitsPerSec(kTargetBitrateBps),
1382 DataRate::BitsPerSec(kTargetBitrateBps),
1383 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001384
perkja49cbd32016-09-16 07:53:41 -07001385 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001386 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001387
mflodmancc3d4422017-08-03 08:27:51 -07001388 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001389 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001390 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001391 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1392 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001393}
1394
mflodmancc3d4422017-08-03 08:27:51 -07001395TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001397 DataRate::BitsPerSec(kTargetBitrateBps),
1398 DataRate::BitsPerSec(kTargetBitrateBps),
1399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001400
1401 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001402 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001403 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001404 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1405 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001406 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1407 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001408 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001409 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001410
mflodmancc3d4422017-08-03 08:27:51 -07001411 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001412}
1413
Noah Richards51db4212019-06-12 06:59:12 -07001414TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001416 DataRate::BitsPerSec(kTargetBitrateBps),
1417 DataRate::BitsPerSec(kTargetBitrateBps),
1418 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001419
1420 rtc::Event frame_destroyed_event;
1421 video_source_.IncomingCapturedFrame(
1422 CreateFakeNativeFrame(1, &frame_destroyed_event));
1423 ExpectDroppedFrame();
1424 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1425 video_stream_encoder_->Stop();
1426}
1427
1428TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1429 // Use the cropping factory.
1430 video_encoder_config_.video_stream_factory =
1431 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, 30);
1432 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1433 kMaxPayloadLength);
1434 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1435
1436 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001438 DataRate::BitsPerSec(kTargetBitrateBps),
1439 DataRate::BitsPerSec(kTargetBitrateBps),
1440 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001441 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1442 WaitForEncodedFrame(1);
1443 // The encoder will have been configured once.
1444 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1445 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1446 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1447
1448 // Now send in a fake frame that needs to be cropped as the width/height
1449 // aren't divisible by 4 (see CreateEncoderStreams above).
1450 rtc::Event frame_destroyed_event;
1451 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1452 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1453 ExpectDroppedFrame();
1454 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1455 video_stream_encoder_->Stop();
1456}
1457
Ying Wang9b881ab2020-02-07 14:29:32 +01001458TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001460 DataRate::BitsPerSec(kTargetBitrateBps),
1461 DataRate::BitsPerSec(kTargetBitrateBps),
1462 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001463 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1464 WaitForEncodedFrame(1);
1465
Henrik Boström381d1092020-05-12 18:49:07 +02001466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001467 DataRate::BitsPerSec(kTargetBitrateBps),
1468 DataRate::BitsPerSec(kTargetBitrateBps),
1469 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001470 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1471 // frames. Adding two frames means that the first frame will be dropped and
1472 // the second frame will be sent to the encoder.
1473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1474 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1475 WaitForEncodedFrame(3);
1476 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1477 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1478 WaitForEncodedFrame(5);
1479 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1480 video_stream_encoder_->Stop();
1481}
1482
mflodmancc3d4422017-08-03 08:27:51 -07001483TEST_F(VideoStreamEncoderTest,
1484 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001486 DataRate::BitsPerSec(kTargetBitrateBps),
1487 DataRate::BitsPerSec(kTargetBitrateBps),
1488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001489 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001490
1491 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001492 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001493 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001494 // The encoder will have been configured once when the first frame is
1495 // received.
1496 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001497
1498 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001499 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001500 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001501 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001502 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001503
1504 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001505 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001506 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001507 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001508 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001509
mflodmancc3d4422017-08-03 08:27:51 -07001510 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001511}
1512
mflodmancc3d4422017-08-03 08:27:51 -07001513TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001515 DataRate::BitsPerSec(kTargetBitrateBps),
1516 DataRate::BitsPerSec(kTargetBitrateBps),
1517 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001518
1519 // Capture a frame and wait for it to synchronize with the encoder thread.
1520 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001521 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001522 // The encoder will have been configured once.
1523 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001524 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1525 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1526
1527 codec_width_ *= 2;
1528 codec_height_ *= 2;
1529 // Capture a frame with a higher resolution and wait for it to synchronize
1530 // with the encoder thread.
1531 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001532 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001533 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1534 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001535 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001536
mflodmancc3d4422017-08-03 08:27:51 -07001537 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001538}
1539
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001540TEST_F(VideoStreamEncoderTest,
1541 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001543 DataRate::BitsPerSec(kTargetBitrateBps),
1544 DataRate::BitsPerSec(kTargetBitrateBps),
1545 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001546
1547 // Capture a frame and wait for it to synchronize with the encoder thread.
1548 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1549 WaitForEncodedFrame(1);
1550
1551 VideoEncoderConfig video_encoder_config;
1552 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1553 // Changing the max payload data length recreates encoder.
1554 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1555 kMaxPayloadLength / 2);
1556
1557 // Capture a frame and wait for it to synchronize with the encoder thread.
1558 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1559 WaitForEncodedFrame(2);
1560 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1561
1562 video_stream_encoder_->Stop();
1563}
1564
Sergey Silkin5ee69672019-07-02 14:18:34 +02001565TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001567 DataRate::BitsPerSec(kTargetBitrateBps),
1568 DataRate::BitsPerSec(kTargetBitrateBps),
1569 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001570
1571 VideoEncoderConfig video_encoder_config;
1572 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1573 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1574 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1575 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1576 kMaxPayloadLength);
1577
1578 // Capture a frame and wait for it to synchronize with the encoder thread.
1579 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1580 WaitForEncodedFrame(1);
1581 // The encoder will have been configured once when the first frame is
1582 // received.
1583 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1584 EXPECT_EQ(kTargetBitrateBps,
1585 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1586 EXPECT_EQ(kStartBitrateBps,
1587 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1588
Sergey Silkin6456e352019-07-08 17:56:40 +02001589 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1590 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001591 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1592 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1593 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1594 kMaxPayloadLength);
1595
1596 // Capture a frame and wait for it to synchronize with the encoder thread.
1597 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1598 WaitForEncodedFrame(2);
1599 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1600 // Bitrate limits have changed - rate allocator should be reconfigured,
1601 // encoder should not be reconfigured.
1602 EXPECT_EQ(kTargetBitrateBps * 2,
1603 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1604 EXPECT_EQ(kStartBitrateBps * 2,
1605 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1606 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1607
1608 video_stream_encoder_->Stop();
1609}
1610
Sergey Silkin6456e352019-07-08 17:56:40 +02001611TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001612 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001614 DataRate::BitsPerSec(kTargetBitrateBps),
1615 DataRate::BitsPerSec(kTargetBitrateBps),
1616 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001617
Sergey Silkincd02eba2020-01-20 14:48:40 +01001618 const uint32_t kMinEncBitrateKbps = 100;
1619 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001620 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001621 /*frame_size_pixels=*/codec_width_ * codec_height_,
1622 /*min_start_bitrate_bps=*/0,
1623 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1624 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001625 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1626
Sergey Silkincd02eba2020-01-20 14:48:40 +01001627 VideoEncoderConfig video_encoder_config;
1628 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1629 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1630 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1631 (kMinEncBitrateKbps + 1) * 1000;
1632 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1633 kMaxPayloadLength);
1634
1635 // When both encoder and app provide bitrate limits, the intersection of
1636 // provided sets should be used.
1637 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1638 WaitForEncodedFrame(1);
1639 EXPECT_EQ(kMaxEncBitrateKbps,
1640 bitrate_allocator_factory_.codec_config().maxBitrate);
1641 EXPECT_EQ(kMinEncBitrateKbps + 1,
1642 bitrate_allocator_factory_.codec_config().minBitrate);
1643
1644 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1645 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1646 (kMinEncBitrateKbps - 1) * 1000;
1647 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1648 kMaxPayloadLength);
1649 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001650 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001651 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001652 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001653 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001654 bitrate_allocator_factory_.codec_config().minBitrate);
1655
Sergey Silkincd02eba2020-01-20 14:48:40 +01001656 video_stream_encoder_->Stop();
1657}
1658
1659TEST_F(VideoStreamEncoderTest,
1660 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001662 DataRate::BitsPerSec(kTargetBitrateBps),
1663 DataRate::BitsPerSec(kTargetBitrateBps),
1664 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001665
1666 const uint32_t kMinAppBitrateKbps = 100;
1667 const uint32_t kMaxAppBitrateKbps = 200;
1668 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1669 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1670 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1671 /*frame_size_pixels=*/codec_width_ * codec_height_,
1672 /*min_start_bitrate_bps=*/0,
1673 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1674 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1675 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1676
1677 VideoEncoderConfig video_encoder_config;
1678 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1679 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1680 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1681 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001682 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1683 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001684
Sergey Silkincd02eba2020-01-20 14:48:40 +01001685 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1686 WaitForEncodedFrame(1);
1687 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001688 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001689 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001690 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001691
1692 video_stream_encoder_->Stop();
1693}
1694
1695TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001696 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001698 DataRate::BitsPerSec(kTargetBitrateBps),
1699 DataRate::BitsPerSec(kTargetBitrateBps),
1700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001701
1702 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001703 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001704 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001705 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001706 fake_encoder_.SetResolutionBitrateLimits(
1707 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1708
1709 VideoEncoderConfig video_encoder_config;
1710 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1711 video_encoder_config.max_bitrate_bps = 0;
1712 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1713 kMaxPayloadLength);
1714
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001715 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001716 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1717 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001718 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1719 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001720 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1721 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1722
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001723 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001724 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1725 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001726 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1727 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001728 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1729 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1730
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001731 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001732 // encoder for 360p should be used.
1733 video_source_.IncomingCapturedFrame(
1734 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1735 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001736 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1737 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001738 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1739 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1740
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001741 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001742 // ignored.
1743 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1744 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001745 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1746 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001747 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1748 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001749 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1750 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001751 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1752 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1753
1754 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1755 // for 270p should be used.
1756 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1757 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001758 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1759 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001760 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1761 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1762
1763 video_stream_encoder_->Stop();
1764}
1765
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001766TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001768 DataRate::BitsPerSec(kTargetBitrateBps),
1769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001771
1772 VideoEncoderConfig video_encoder_config;
1773 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1774 video_encoder_config.max_bitrate_bps = 0;
1775 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1776 kMaxPayloadLength);
1777
1778 // Encode 720p frame to get the default encoder target bitrate.
1779 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1780 WaitForEncodedFrame(1);
1781 const uint32_t kDefaultTargetBitrateFor720pKbps =
1782 bitrate_allocator_factory_.codec_config()
1783 .simulcastStream[0]
1784 .targetBitrate;
1785
1786 // Set the max recommended encoder bitrate to something lower than the default
1787 // target bitrate.
1788 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1789 1280 * 720, 10 * 1000, 10 * 1000,
1790 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1791 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1792
1793 // Change resolution to trigger encoder reinitialization.
1794 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1795 WaitForEncodedFrame(2);
1796 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1797 WaitForEncodedFrame(3);
1798
1799 // Ensure the target bitrate is capped by the max bitrate.
1800 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1801 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1802 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1803 .simulcastStream[0]
1804 .targetBitrate *
1805 1000,
1806 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1807
1808 video_stream_encoder_->Stop();
1809}
1810
mflodmancc3d4422017-08-03 08:27:51 -07001811TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07001812 EXPECT_TRUE(video_source_.has_sinks());
1813 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07001814 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001815 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07001816 EXPECT_FALSE(video_source_.has_sinks());
1817 EXPECT_TRUE(new_video_source.has_sinks());
1818
mflodmancc3d4422017-08-03 08:27:51 -07001819 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001820}
1821
mflodmancc3d4422017-08-03 08:27:51 -07001822TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07001823 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001824 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07001825 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07001826 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07001827}
1828
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001829TEST_F(VideoStreamEncoderTest, SinkWantsResolutionAlignment) {
1830 constexpr int kRequestedResolutionAlignment = 7;
1831 video_source_.set_adaptation_enabled(true);
1832 fake_encoder_.SetRequestedResolutionAlignment(kRequestedResolutionAlignment);
Henrik Boström381d1092020-05-12 18:49:07 +02001833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001834 DataRate::BitsPerSec(kTargetBitrateBps),
1835 DataRate::BitsPerSec(kTargetBitrateBps),
1836 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001837
1838 // On the 1st frame, we should have initialized the encoder and
1839 // asked for its resolution requirements.
1840 video_source_.IncomingCapturedFrame(
1841 CreateFrame(1, codec_width_, codec_height_));
1842 WaitForEncodedFrame(1);
1843 EXPECT_EQ(video_source_.sink_wants().resolution_alignment,
1844 kRequestedResolutionAlignment);
1845
1846 // On the 2nd frame, we should be receiving a correctly aligned resolution.
1847 // (It's up the to the encoder to potentially drop the previous frame,
1848 // to avoid coding back-to-back keyframes.)
1849 video_source_.IncomingCapturedFrame(
1850 CreateFrame(2, codec_width_, codec_height_));
1851 WaitForEncodedFrame(2);
1852 sink_.CheckLastFrameSizeIsMultipleOf(kRequestedResolutionAlignment);
1853
1854 video_stream_encoder_->Stop();
1855}
1856
Jonathan Yubc771b72017-12-08 17:04:29 -08001857TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
1858 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07001859 const int kWidth = 1280;
1860 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08001861
1862 // We rely on the automatic resolution adaptation, but we handle framerate
1863 // adaptation manually by mocking the stats proxy.
1864 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07001865
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001866 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02001867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001868 DataRate::BitsPerSec(kTargetBitrateBps),
1869 DataRate::BitsPerSec(kTargetBitrateBps),
1870 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07001871 video_stream_encoder_->SetSource(&video_source_,
1872 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001873 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07001874 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001875 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07001876 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
1877
Jonathan Yubc771b72017-12-08 17:04:29 -08001878 // Adapt down as far as possible.
1879 rtc::VideoSinkWants last_wants;
1880 int64_t t = 1;
1881 int loop_count = 0;
1882 do {
1883 ++loop_count;
1884 last_wants = video_source_.sink_wants();
1885
1886 // Simulate the framerate we've been asked to adapt to.
1887 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1888 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1889 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1890 mock_stats.input_frame_rate = fps;
1891 stats_proxy_->SetMockStats(mock_stats);
1892
1893 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1894 sink_.WaitForEncodedFrame(t);
1895 t += frame_interval_ms;
1896
mflodmancc3d4422017-08-03 08:27:51 -07001897 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001898 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001899 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001900 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1901 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001902 } while (video_source_.sink_wants().max_pixel_count <
1903 last_wants.max_pixel_count ||
1904 video_source_.sink_wants().max_framerate_fps <
1905 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001906
Jonathan Yubc771b72017-12-08 17:04:29 -08001907 // Verify that we've adapted all the way down.
1908 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001909 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001910 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
1911 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07001912 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08001913 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
1914 *video_source_.last_sent_height());
1915 EXPECT_EQ(kMinBalancedFramerateFps,
1916 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001917
Jonathan Yubc771b72017-12-08 17:04:29 -08001918 // Adapt back up the same number of times we adapted down.
1919 for (int i = 0; i < loop_count - 1; ++i) {
1920 last_wants = video_source_.sink_wants();
1921
1922 // Simulate the framerate we've been asked to adapt to.
1923 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
1924 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
1925 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
1926 mock_stats.input_frame_rate = fps;
1927 stats_proxy_->SetMockStats(mock_stats);
1928
1929 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
1930 sink_.WaitForEncodedFrame(t);
1931 t += frame_interval_ms;
1932
Henrik Boström91aa7322020-04-28 12:24:33 +02001933 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001934 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08001935 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001936 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
1937 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08001938 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
1939 last_wants.max_pixel_count ||
1940 video_source_.sink_wants().max_framerate_fps >
1941 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07001942 }
1943
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001944 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08001945 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07001946 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08001947 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
1948 EXPECT_EQ((loop_count - 1) * 2,
1949 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07001950
mflodmancc3d4422017-08-03 08:27:51 -07001951 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07001952}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001953
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001954TEST_F(VideoStreamEncoderTest,
1955 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
1956 video_stream_encoder_->OnBitrateUpdated(
1957 DataRate::BitsPerSec(kTargetBitrateBps),
1958 DataRate::BitsPerSec(kTargetBitrateBps),
1959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001960 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001961
1962 const int kFrameWidth = 1280;
1963 const int kFrameHeight = 720;
1964
1965 int64_t ntp_time = kFrameIntervalMs;
1966
1967 // Force an input frame rate to be available, or the adaptation call won't
1968 // know what framerate to adapt form.
1969 const int kInputFps = 30;
1970 VideoSendStream::Stats stats = stats_proxy_->GetStats();
1971 stats.input_frame_rate = kInputFps;
1972 stats_proxy_->SetMockStats(stats);
1973
1974 video_source_.set_adaptation_enabled(true);
1975 video_stream_encoder_->SetSource(
1976 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02001977 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02001978 video_source_.IncomingCapturedFrame(
1979 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1980 sink_.WaitForEncodedFrame(ntp_time);
1981 ntp_time += kFrameIntervalMs;
1982
1983 // Trigger CPU overuse.
1984 video_stream_encoder_->TriggerCpuOveruse();
1985 video_source_.IncomingCapturedFrame(
1986 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1987 sink_.WaitForEncodedFrame(ntp_time);
1988 ntp_time += kFrameIntervalMs;
1989
1990 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
1991 EXPECT_EQ(std::numeric_limits<int>::max(),
1992 video_source_.sink_wants().max_pixel_count);
1993 // Some framerate constraint should be set.
1994 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
1995 EXPECT_LT(restricted_fps, kInputFps);
1996 video_source_.IncomingCapturedFrame(
1997 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
1998 sink_.WaitForEncodedFrame(ntp_time);
1999 ntp_time += 100;
2000
Henrik Boström2671dac2020-05-19 16:29:09 +02002001 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002002 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2003 // Give the encoder queue time to process the change in degradation preference
2004 // by waiting for an encoded frame.
2005 video_source_.IncomingCapturedFrame(
2006 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2007 sink_.WaitForEncodedFrame(ntp_time);
2008 ntp_time += kFrameIntervalMs;
2009
2010 video_stream_encoder_->TriggerQualityLow();
2011 video_source_.IncomingCapturedFrame(
2012 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2013 sink_.WaitForEncodedFrame(ntp_time);
2014 ntp_time += kFrameIntervalMs;
2015
2016 // Some resolution constraint should be set.
2017 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2018 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2019 kFrameWidth * kFrameHeight);
2020 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2021
2022 int pixel_count = video_source_.sink_wants().max_pixel_count;
2023 // Triggering a CPU underuse should not change the sink wants since it has
2024 // not been overused for resolution since we changed degradation preference.
2025 video_stream_encoder_->TriggerCpuUnderuse();
2026 video_source_.IncomingCapturedFrame(
2027 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2028 sink_.WaitForEncodedFrame(ntp_time);
2029 ntp_time += kFrameIntervalMs;
2030 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2031 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2032
Evan Shrubsole64469032020-06-11 10:45:29 +02002033 // Change the degradation preference back. CPU underuse should not adapt since
2034 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002035 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002036 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2037 video_source_.IncomingCapturedFrame(
2038 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2039 sink_.WaitForEncodedFrame(ntp_time);
2040 ntp_time += 100;
2041 // Resolution adaptations is gone after changing degradation preference.
2042 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2043 EXPECT_EQ(std::numeric_limits<int>::max(),
2044 video_source_.sink_wants().max_pixel_count);
2045 // The fps adaptation from above is now back.
2046 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2047
2048 // Trigger CPU underuse.
2049 video_stream_encoder_->TriggerCpuUnderuse();
2050 video_source_.IncomingCapturedFrame(
2051 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2052 sink_.WaitForEncodedFrame(ntp_time);
2053 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002054 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2055
2056 // Trigger QP underuse, fps should return to normal.
2057 video_stream_encoder_->TriggerQualityHigh();
2058 video_source_.IncomingCapturedFrame(
2059 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2060 sink_.WaitForEncodedFrame(ntp_time);
2061 ntp_time += kFrameIntervalMs;
2062 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002063
2064 video_stream_encoder_->Stop();
2065}
2066
mflodmancc3d4422017-08-03 08:27:51 -07002067TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002069 DataRate::BitsPerSec(kTargetBitrateBps),
2070 DataRate::BitsPerSec(kTargetBitrateBps),
2071 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002072 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002073
sprangc5d62e22017-04-02 23:53:04 -07002074 const int kFrameWidth = 1280;
2075 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002076
Åsa Persson8c1bf952018-09-13 10:42:19 +02002077 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002078
kthelgason5e13d412016-12-01 03:59:51 -08002079 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002080 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002081 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002082 frame_timestamp += kFrameIntervalMs;
2083
perkj803d97f2016-11-01 11:45:46 -07002084 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002085 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002086 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002087 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002088 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002089 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002090
asapersson0944a802017-04-07 00:57:58 -07002091 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002092 // wanted resolution.
2093 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2094 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2095 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002096 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002097
2098 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002099 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002100 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002101 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002102 // Give the encoder queue time to process the change in degradation preference
2103 // by waiting for an encoded frame.
2104 new_video_source.IncomingCapturedFrame(
2105 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2106 sink_.WaitForEncodedFrame(frame_timestamp);
2107 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002108 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002109 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002110
sprangc5d62e22017-04-02 23:53:04 -07002111 // Force an input frame rate to be available, or the adaptation call won't
2112 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002113 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002114 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002115 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002116 stats_proxy_->SetMockStats(stats);
2117
mflodmancc3d4422017-08-03 08:27:51 -07002118 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002119 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002120 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002121 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002122 frame_timestamp += kFrameIntervalMs;
2123
2124 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002125 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002126 EXPECT_EQ(std::numeric_limits<int>::max(),
2127 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002128 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002129
asapersson02465b82017-04-10 01:12:52 -07002130 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002131 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2132 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002133 // Give the encoder queue time to process the change in degradation preference
2134 // by waiting for an encoded frame.
2135 new_video_source.IncomingCapturedFrame(
2136 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2137 sink_.WaitForEncodedFrame(frame_timestamp);
2138 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002139 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002140
mflodmancc3d4422017-08-03 08:27:51 -07002141 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002142 new_video_source.IncomingCapturedFrame(
2143 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002144 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002145 frame_timestamp += kFrameIntervalMs;
2146
2147 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002148 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002149
2150 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002151 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002152 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002153 // Give the encoder queue time to process the change in degradation preference
2154 // by waiting for an encoded frame.
2155 new_video_source.IncomingCapturedFrame(
2156 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2157 sink_.WaitForEncodedFrame(frame_timestamp);
2158 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002159 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2160 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002161 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002162 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002163
2164 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002165 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002166 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002167 // Give the encoder queue time to process the change in degradation preference
2168 // by waiting for an encoded frame.
2169 new_video_source.IncomingCapturedFrame(
2170 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2171 sink_.WaitForEncodedFrame(frame_timestamp);
2172 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002173 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2174 EXPECT_EQ(std::numeric_limits<int>::max(),
2175 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002176 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002177
mflodmancc3d4422017-08-03 08:27:51 -07002178 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002179}
2180
mflodmancc3d4422017-08-03 08:27:51 -07002181TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002183 DataRate::BitsPerSec(kTargetBitrateBps),
2184 DataRate::BitsPerSec(kTargetBitrateBps),
2185 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002186
asaperssonfab67072017-04-04 05:51:49 -07002187 const int kWidth = 1280;
2188 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002189 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002190 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002191 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2192 EXPECT_FALSE(stats.bw_limited_resolution);
2193 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2194
2195 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002196 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002197 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002198 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002199
2200 stats = stats_proxy_->GetStats();
2201 EXPECT_TRUE(stats.bw_limited_resolution);
2202 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2203
2204 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002205 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002206 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002207 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002208
2209 stats = stats_proxy_->GetStats();
2210 EXPECT_FALSE(stats.bw_limited_resolution);
2211 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2212 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2213
mflodmancc3d4422017-08-03 08:27:51 -07002214 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002215}
2216
mflodmancc3d4422017-08-03 08:27:51 -07002217TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002219 DataRate::BitsPerSec(kTargetBitrateBps),
2220 DataRate::BitsPerSec(kTargetBitrateBps),
2221 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002222
2223 const int kWidth = 1280;
2224 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002225 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002226 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002227 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2228 EXPECT_FALSE(stats.cpu_limited_resolution);
2229 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2230
2231 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002232 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002233 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002234 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002235
2236 stats = stats_proxy_->GetStats();
2237 EXPECT_TRUE(stats.cpu_limited_resolution);
2238 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2239
2240 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002241 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002242 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002243 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002244
2245 stats = stats_proxy_->GetStats();
2246 EXPECT_FALSE(stats.cpu_limited_resolution);
2247 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002248 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002249
mflodmancc3d4422017-08-03 08:27:51 -07002250 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002251}
2252
mflodmancc3d4422017-08-03 08:27:51 -07002253TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002255 DataRate::BitsPerSec(kTargetBitrateBps),
2256 DataRate::BitsPerSec(kTargetBitrateBps),
2257 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002258
asaperssonfab67072017-04-04 05:51:49 -07002259 const int kWidth = 1280;
2260 const int kHeight = 720;
2261 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002262 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002263 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002264 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002265 EXPECT_FALSE(stats.cpu_limited_resolution);
2266 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2267
asaperssonfab67072017-04-04 05:51:49 -07002268 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002269 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002270 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002271 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002272 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002273 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002274 EXPECT_TRUE(stats.cpu_limited_resolution);
2275 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2276
2277 // Set new source with adaptation still enabled.
2278 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002280 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002281
asaperssonfab67072017-04-04 05:51:49 -07002282 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002283 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002284 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002285 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002286 EXPECT_TRUE(stats.cpu_limited_resolution);
2287 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2288
2289 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002290 video_stream_encoder_->SetSource(&new_video_source,
2291 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002292
asaperssonfab67072017-04-04 05:51:49 -07002293 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002294 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002295 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002296 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002297 EXPECT_FALSE(stats.cpu_limited_resolution);
2298 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2299
2300 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002301 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002302 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002303
asaperssonfab67072017-04-04 05:51:49 -07002304 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002305 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002306 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002307 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002308 EXPECT_TRUE(stats.cpu_limited_resolution);
2309 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2310
asaperssonfab67072017-04-04 05:51:49 -07002311 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002312 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002313 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002314 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002315 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002316 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002317 EXPECT_FALSE(stats.cpu_limited_resolution);
2318 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002319 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002320
mflodmancc3d4422017-08-03 08:27:51 -07002321 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002322}
2323
mflodmancc3d4422017-08-03 08:27:51 -07002324TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002326 DataRate::BitsPerSec(kTargetBitrateBps),
2327 DataRate::BitsPerSec(kTargetBitrateBps),
2328 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002329
asaperssonfab67072017-04-04 05:51:49 -07002330 const int kWidth = 1280;
2331 const int kHeight = 720;
2332 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002333 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002334 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002335 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002336 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002337 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002338
2339 // Set new source with adaptation still enabled.
2340 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002341 video_stream_encoder_->SetSource(&new_video_source,
2342 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002343
asaperssonfab67072017-04-04 05:51:49 -07002344 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002345 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002346 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002347 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002348 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002349 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002350
asaperssonfab67072017-04-04 05:51:49 -07002351 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002352 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002353 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002354 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002355 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002356 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002357 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002358 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002359
asaperssonfab67072017-04-04 05:51:49 -07002360 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002361 video_stream_encoder_->SetSource(&new_video_source,
2362 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002363
asaperssonfab67072017-04-04 05:51:49 -07002364 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002365 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002366 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002367 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002368 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002369 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002370
asapersson02465b82017-04-10 01:12:52 -07002371 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002373 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002374
asaperssonfab67072017-04-04 05:51:49 -07002375 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002376 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002377 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002378 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002379 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002380 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2381 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002382
mflodmancc3d4422017-08-03 08:27:51 -07002383 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002384}
2385
mflodmancc3d4422017-08-03 08:27:51 -07002386TEST_F(VideoStreamEncoderTest,
2387 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002389 DataRate::BitsPerSec(kTargetBitrateBps),
2390 DataRate::BitsPerSec(kTargetBitrateBps),
2391 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002392
2393 const int kWidth = 1280;
2394 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002395 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002396 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002397 video_source_.IncomingCapturedFrame(
2398 CreateFrame(timestamp_ms, kWidth, kHeight));
2399 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002400 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2402 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2403
2404 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002406 timestamp_ms += kFrameIntervalMs;
2407 video_source_.IncomingCapturedFrame(
2408 CreateFrame(timestamp_ms, kWidth, kHeight));
2409 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002410 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2411 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2412 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2413
2414 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002415 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002416 timestamp_ms += kFrameIntervalMs;
2417 video_source_.IncomingCapturedFrame(
2418 CreateFrame(timestamp_ms, kWidth, kHeight));
2419 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002420 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2421 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2422 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2423
Niels Möller4db138e2018-04-19 09:04:13 +02002424 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002425 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002426
2427 VideoEncoderConfig video_encoder_config;
2428 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2429 // Make format different, to force recreation of encoder.
2430 video_encoder_config.video_format.parameters["foo"] = "foo";
2431 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002432 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002433 timestamp_ms += kFrameIntervalMs;
2434 video_source_.IncomingCapturedFrame(
2435 CreateFrame(timestamp_ms, kWidth, kHeight));
2436 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002437 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2438 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2439 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2440
mflodmancc3d4422017-08-03 08:27:51 -07002441 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002442}
2443
mflodmancc3d4422017-08-03 08:27:51 -07002444TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002445 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002447 DataRate::BitsPerSec(kTargetBitrateBps),
2448 DataRate::BitsPerSec(kTargetBitrateBps),
2449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2450
2451 const int kWidth = 1280;
2452 const int kHeight = 720;
2453 int sequence = 1;
2454
2455 // Enable BALANCED preference, no initial limitation.
2456 test::FrameForwarder source;
2457 video_stream_encoder_->SetSource(&source,
2458 webrtc::DegradationPreference::BALANCED);
2459 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2460 WaitForEncodedFrame(sequence++);
2461 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2462 EXPECT_FALSE(stats.cpu_limited_resolution);
2463 EXPECT_FALSE(stats.cpu_limited_framerate);
2464 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2465
2466 // Trigger CPU overuse, should now adapt down.
2467 video_stream_encoder_->TriggerCpuOveruse();
2468 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2469 WaitForEncodedFrame(sequence++);
2470 stats = stats_proxy_->GetStats();
2471 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2472
2473 // Set new degradation preference should clear restrictions since we changed
2474 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002475 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002476 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2477 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2478 WaitForEncodedFrame(sequence++);
2479 stats = stats_proxy_->GetStats();
2480 EXPECT_FALSE(stats.cpu_limited_resolution);
2481 EXPECT_FALSE(stats.cpu_limited_framerate);
2482 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2483
2484 // Force an input frame rate to be available, or the adaptation call won't
2485 // know what framerate to adapt from.
2486 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2487 mock_stats.input_frame_rate = 30;
2488 stats_proxy_->SetMockStats(mock_stats);
2489 video_stream_encoder_->TriggerCpuOveruse();
2490 stats_proxy_->ResetMockStats();
2491 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2492 WaitForEncodedFrame(sequence++);
2493
2494 // We have now adapted once.
2495 stats = stats_proxy_->GetStats();
2496 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2497
2498 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002499 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2500 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002501 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2502 WaitForEncodedFrame(sequence++);
2503 stats = stats_proxy_->GetStats();
2504 EXPECT_FALSE(stats.cpu_limited_resolution);
2505 EXPECT_FALSE(stats.cpu_limited_framerate);
2506 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2507
2508 video_stream_encoder_->Stop();
2509}
2510
2511TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002512 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002514 DataRate::BitsPerSec(kTargetBitrateBps),
2515 DataRate::BitsPerSec(kTargetBitrateBps),
2516 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002517
asapersson0944a802017-04-07 00:57:58 -07002518 const int kWidth = 1280;
2519 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002520 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002521
asaperssonfab67072017-04-04 05:51:49 -07002522 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002523 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002524 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002525 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002526 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002527 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2528
asapersson02465b82017-04-10 01:12:52 -07002529 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002530 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002531 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002532 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002533 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002534 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002535 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002536 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2537
2538 // Set new source with adaptation still enabled.
2539 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002540 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002541 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002542
2543 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002544 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002545 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002546 stats = stats_proxy_->GetStats();
2547 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002548 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002549 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2550
sprangc5d62e22017-04-02 23:53:04 -07002551 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07002552 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07002554 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002555 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002556 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002557 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07002558 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07002559 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07002560 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002561 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2562
sprangc5d62e22017-04-02 23:53:04 -07002563 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07002564 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07002565 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2566 mock_stats.input_frame_rate = 30;
2567 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002568 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002569 stats_proxy_->ResetMockStats();
2570
2571 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002572 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002573 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002574
2575 // Framerate now adapted.
2576 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002577 EXPECT_FALSE(stats.cpu_limited_resolution);
2578 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002579 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2580
2581 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002582 video_stream_encoder_->SetSource(&new_video_source,
2583 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07002584 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002585 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002586 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002587
2588 stats = stats_proxy_->GetStats();
2589 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002590 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002591 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2592
2593 // Try to trigger overuse. Should not succeed.
2594 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07002595 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002596 stats_proxy_->ResetMockStats();
2597
2598 stats = stats_proxy_->GetStats();
2599 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002600 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002601 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2602
2603 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002604 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002605 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07002606 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002607 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002608 stats = stats_proxy_->GetStats();
2609 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002610 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002611 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002612
2613 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002614 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002615 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002616 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002617 stats = stats_proxy_->GetStats();
2618 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002619 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002620 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2621
2622 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002623 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002624 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07002625 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002626 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002627 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002628 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07002629 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07002630 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002631 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002632 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
2633
2634 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02002635 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07002636 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002637 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002638 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07002639 stats = stats_proxy_->GetStats();
2640 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002641 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07002642 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002643 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002644
mflodmancc3d4422017-08-03 08:27:51 -07002645 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002646}
2647
mflodmancc3d4422017-08-03 08:27:51 -07002648TEST_F(VideoStreamEncoderTest,
2649 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07002650 const int kWidth = 1280;
2651 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002653 DataRate::BitsPerSec(kTargetBitrateBps),
2654 DataRate::BitsPerSec(kTargetBitrateBps),
2655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002656
asaperssonfab67072017-04-04 05:51:49 -07002657 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002658 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08002659
asaperssonfab67072017-04-04 05:51:49 -07002660 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002661 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002662
asaperssonfab67072017-04-04 05:51:49 -07002663 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002664 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08002665
asaperssonfab67072017-04-04 05:51:49 -07002666 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002667 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08002668
kthelgason876222f2016-11-29 01:44:11 -08002669 // Expect a scale down.
2670 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07002671 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08002672
asapersson02465b82017-04-10 01:12:52 -07002673 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08002674 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002675 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002676 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002677
asaperssonfab67072017-04-04 05:51:49 -07002678 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002680 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002681 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002682
asaperssonfab67072017-04-04 05:51:49 -07002683 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002684 EXPECT_EQ(std::numeric_limits<int>::max(),
2685 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002686
asaperssonfab67072017-04-04 05:51:49 -07002687 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07002688 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002689 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002690 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002691
asapersson02465b82017-04-10 01:12:52 -07002692 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07002693 EXPECT_EQ(std::numeric_limits<int>::max(),
2694 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08002695
mflodmancc3d4422017-08-03 08:27:51 -07002696 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002697}
2698
mflodmancc3d4422017-08-03 08:27:51 -07002699TEST_F(VideoStreamEncoderTest,
2700 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002701 const int kWidth = 1280;
2702 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002704 DataRate::BitsPerSec(kTargetBitrateBps),
2705 DataRate::BitsPerSec(kTargetBitrateBps),
2706 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002707
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002708 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002709 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002710 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002711 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002712
2713 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002714 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002715 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002716 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2718
2719 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002720 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002721 EXPECT_THAT(source.sink_wants(),
2722 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002723 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2724 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2726
2727 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002728 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07002729 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2730 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2731 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2732
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002734}
2735
mflodmancc3d4422017-08-03 08:27:51 -07002736TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002737 const int kWidth = 1280;
2738 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002740 DataRate::BitsPerSec(kTargetBitrateBps),
2741 DataRate::BitsPerSec(kTargetBitrateBps),
2742 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002743
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002744 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002745 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002746 video_stream_encoder_->SetSource(&source,
2747 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002748 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2749 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002750 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002751
2752 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002753 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002754 EXPECT_THAT(source.sink_wants(),
2755 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07002756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2757 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2758 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
2759
2760 // Trigger adapt down for same input resolution, expect no change.
2761 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2762 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07002763 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002764 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2766 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2767
2768 // Trigger adapt down for larger input resolution, expect no change.
2769 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
2770 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07002771 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07002772 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
2773 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2774 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2775
mflodmancc3d4422017-08-03 08:27:51 -07002776 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002777}
2778
mflodmancc3d4422017-08-03 08:27:51 -07002779TEST_F(VideoStreamEncoderTest,
2780 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002781 const int kWidth = 1280;
2782 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002783 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002784 DataRate::BitsPerSec(kTargetBitrateBps),
2785 DataRate::BitsPerSec(kTargetBitrateBps),
2786 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002787
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002788 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002789 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002790 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002791 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002792
2793 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002794 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002796 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2797 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2798
2799 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002800 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002801 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2803 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2804
mflodmancc3d4422017-08-03 08:27:51 -07002805 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002806}
2807
mflodmancc3d4422017-08-03 08:27:51 -07002808TEST_F(VideoStreamEncoderTest,
2809 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07002810 const int kWidth = 1280;
2811 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002813 DataRate::BitsPerSec(kTargetBitrateBps),
2814 DataRate::BitsPerSec(kTargetBitrateBps),
2815 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002816
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002817 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002818 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002820 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07002821
2822 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002823 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002824 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002825 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002826 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2827
2828 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02002829 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002830 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002831 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07002832 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2833
mflodmancc3d4422017-08-03 08:27:51 -07002834 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002835}
2836
mflodmancc3d4422017-08-03 08:27:51 -07002837TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07002838 const int kWidth = 1280;
2839 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002841 DataRate::BitsPerSec(kTargetBitrateBps),
2842 DataRate::BitsPerSec(kTargetBitrateBps),
2843 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07002844
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002845 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07002846 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002847 video_stream_encoder_->SetSource(&source,
2848 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07002849
2850 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2851 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002852 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2855 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2856
2857 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002858 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002859 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07002860 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2861 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2862 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2863
mflodmancc3d4422017-08-03 08:27:51 -07002864 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002865}
2866
mflodmancc3d4422017-08-03 08:27:51 -07002867TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07002868 const int kWidth = 1280;
2869 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002871 DataRate::BitsPerSec(kTargetBitrateBps),
2872 DataRate::BitsPerSec(kTargetBitrateBps),
2873 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002874
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002875 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07002876 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002877 video_stream_encoder_->SetSource(&source,
2878 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07002879
2880 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2881 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002882 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2884 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2885 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2886
2887 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07002888 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002889 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002890 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2891 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2893
mflodmancc3d4422017-08-03 08:27:51 -07002894 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002895}
2896
mflodmancc3d4422017-08-03 08:27:51 -07002897TEST_F(VideoStreamEncoderTest,
2898 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07002899 const int kWidth = 1280;
2900 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02002901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002902 DataRate::BitsPerSec(kTargetBitrateBps),
2903 DataRate::BitsPerSec(kTargetBitrateBps),
2904 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07002905
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002906 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07002907 AdaptingFrameForwarder source;
2908 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002910 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07002911
2912 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002914 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002915 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2916 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2917
2918 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002919 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07002920 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002922 EXPECT_THAT(source.sink_wants(),
2923 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07002924 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2925 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2926
2927 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002929 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07002930 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2931 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
2932 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2933
mflodmancc3d4422017-08-03 08:27:51 -07002934 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07002935}
2936
mflodmancc3d4422017-08-03 08:27:51 -07002937TEST_F(VideoStreamEncoderTest,
2938 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07002939 const int kWidth = 1280;
2940 const int kHeight = 720;
2941 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02002942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002943 DataRate::BitsPerSec(kTargetBitrateBps),
2944 DataRate::BitsPerSec(kTargetBitrateBps),
2945 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07002946
2947 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2948 stats.input_frame_rate = kInputFps;
2949 stats_proxy_->SetMockStats(stats);
2950
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002951 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07002952 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
2953 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002954 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002955
2956 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07002957 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07002958 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
2959 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002960 EXPECT_THAT(video_source_.sink_wants(),
2961 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07002962
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002963 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07002964 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02002965 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002966 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002967 // Give the encoder queue time to process the change in degradation preference
2968 // by waiting for an encoded frame.
2969 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
2970 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002971 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002972
2973 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07002974 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01002975 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
2976 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002977 EXPECT_THAT(new_video_source.sink_wants(),
2978 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07002979
2980 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002982 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07002983
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07002985}
2986
mflodmancc3d4422017-08-03 08:27:51 -07002987TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07002988 const int kWidth = 1280;
2989 const int kHeight = 720;
2990 const size_t kNumFrames = 10;
2991
Henrik Boström381d1092020-05-12 18:49:07 +02002992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002993 DataRate::BitsPerSec(kTargetBitrateBps),
2994 DataRate::BitsPerSec(kTargetBitrateBps),
2995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08002996
asaperssond0de2952017-04-21 01:47:31 -07002997 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07002998 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07002999 video_source_.set_adaptation_enabled(true);
3000
3001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3002 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3003
3004 int downscales = 0;
3005 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003006 video_source_.IncomingCapturedFrame(
3007 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3008 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003009
asaperssonfab67072017-04-04 05:51:49 -07003010 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003011 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003012 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003013 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003014
3015 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3016 ++downscales;
3017
3018 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3019 EXPECT_EQ(downscales,
3020 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3021 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003022 }
mflodmancc3d4422017-08-03 08:27:51 -07003023 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003024}
3025
mflodmancc3d4422017-08-03 08:27:51 -07003026TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003027 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3028 const int kWidth = 1280;
3029 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003031 DataRate::BitsPerSec(kTargetBitrateBps),
3032 DataRate::BitsPerSec(kTargetBitrateBps),
3033 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003034
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003035 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003036 AdaptingFrameForwarder source;
3037 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003038 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003039 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003040
Åsa Persson8c1bf952018-09-13 10:42:19 +02003041 int64_t timestamp_ms = kFrameIntervalMs;
3042 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003043 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003044 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003045 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3046 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3047
3048 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003050 timestamp_ms += kFrameIntervalMs;
3051 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3052 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003053 EXPECT_THAT(source.sink_wants(),
3054 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003055 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3056 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3057
3058 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003059 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003060 timestamp_ms += kFrameIntervalMs;
3061 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003062 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003063 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003064 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3065 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3066
3067 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003068 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003069 timestamp_ms += kFrameIntervalMs;
3070 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3071 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003072 EXPECT_THAT(source.sink_wants(),
3073 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003074 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3075 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3076
3077 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003078 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003079 timestamp_ms += kFrameIntervalMs;
3080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003081 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003082 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003083 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3084 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3085
mflodmancc3d4422017-08-03 08:27:51 -07003086 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003087}
3088
mflodmancc3d4422017-08-03 08:27:51 -07003089TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003090 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3091 const int kWidth = 1280;
3092 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003094 DataRate::BitsPerSec(kTargetBitrateBps),
3095 DataRate::BitsPerSec(kTargetBitrateBps),
3096 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003097
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003098 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003099 AdaptingFrameForwarder source;
3100 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003101 video_stream_encoder_->SetSource(&source,
3102 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003103
Åsa Persson8c1bf952018-09-13 10:42:19 +02003104 int64_t timestamp_ms = kFrameIntervalMs;
3105 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003106 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003107 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003108 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3109 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3110
3111 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003113 timestamp_ms += kFrameIntervalMs;
3114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3115 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003116 EXPECT_THAT(source.sink_wants(),
3117 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003118 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3119 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3120
3121 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003122 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003123 timestamp_ms += kFrameIntervalMs;
3124 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003125 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003126 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003127 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3128 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3129
3130 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003131 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003132 timestamp_ms += kFrameIntervalMs;
3133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3134 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003135 EXPECT_THAT(source.sink_wants(),
3136 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003137 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3138 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3139
3140 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003141 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003142 timestamp_ms += kFrameIntervalMs;
3143 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003144 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003145 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003146 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3147 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3148
mflodmancc3d4422017-08-03 08:27:51 -07003149 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003150}
3151
Sergey Silkin41c650b2019-10-14 13:12:19 +02003152TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3153 fake_encoder_.SetResolutionBitrateLimits(
3154 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3155
Henrik Boström381d1092020-05-12 18:49:07 +02003156 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003157 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3158 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3159 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3160 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003161
3162 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3163 AdaptingFrameForwarder source;
3164 source.set_adaptation_enabled(true);
3165 video_stream_encoder_->SetSource(
3166 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3167
3168 // Insert 720p frame.
3169 int64_t timestamp_ms = kFrameIntervalMs;
3170 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3171 WaitForEncodedFrame(1280, 720);
3172
3173 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003175 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3176 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3177 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3178 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003179 video_stream_encoder_->TriggerQualityLow();
3180
3181 // Insert 720p frame. It should be downscaled and encoded.
3182 timestamp_ms += kFrameIntervalMs;
3183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3184 WaitForEncodedFrame(960, 540);
3185
3186 // Trigger adapt up. Higher resolution should not be requested duo to lack
3187 // of bitrate.
3188 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003189 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003190
3191 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003193 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3194 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3195 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3196 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003197
3198 // Trigger adapt up. Higher resolution should be requested.
3199 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003200 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003201
3202 video_stream_encoder_->Stop();
3203}
3204
3205TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3206 fake_encoder_.SetResolutionBitrateLimits(
3207 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3208
3209 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003211 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3212 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3213 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3214 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003215
3216 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
3217 AdaptingFrameForwarder source;
3218 source.set_adaptation_enabled(true);
3219 video_stream_encoder_->SetSource(
3220 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3221
3222 // Insert 720p frame. It should be dropped and lower resolution should be
3223 // requested.
3224 int64_t timestamp_ms = kFrameIntervalMs;
3225 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3226 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003227 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003228
3229 // Insert 720p frame. It should be downscaled and encoded.
3230 timestamp_ms += kFrameIntervalMs;
3231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3232 WaitForEncodedFrame(960, 540);
3233
3234 video_stream_encoder_->Stop();
3235}
3236
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003237class BalancedDegradationTest : public VideoStreamEncoderTest {
3238 protected:
3239 void SetupTest() {
3240 // Reset encoder for field trials to take effect.
3241 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003242 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003243
3244 // Enable BALANCED preference.
3245 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003246 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3247 }
3248
3249 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003250 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003251 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3252 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003253 }
3254
Åsa Persson45b176f2019-09-30 11:19:05 +02003255 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003256 timestamp_ms_ += kFrameIntervalMs;
3257 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003258 }
3259
3260 void InsertFrameAndWaitForEncoded() {
3261 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003262 sink_.WaitForEncodedFrame(timestamp_ms_);
3263 }
3264
3265 const int kWidth = 640; // pixels:640x360=230400
3266 const int kHeight = 360;
3267 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3268 int64_t timestamp_ms_ = 0;
3269 AdaptingFrameForwarder source_;
3270};
3271
3272TEST_F(BalancedDegradationTest, AdaptDownReturnsFalseIfFpsDiffLtThreshold) {
3273 test::ScopedFieldTrials field_trials(
3274 "WebRTC-Video-BalancedDegradationSettings/"
3275 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3276 SetupTest();
3277
3278 // Force input frame rate.
3279 const int kInputFps = 24;
3280 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3281 stats.input_frame_rate = kInputFps;
3282 stats_proxy_->SetMockStats(stats);
3283
Åsa Persson45b176f2019-09-30 11:19:05 +02003284 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003285 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003286
3287 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003288 // Fps diff (input-requested:0) < threshold, expect adapting down not to clear
3289 // QP samples.
3290 EXPECT_FALSE(
3291 video_stream_encoder_
3292 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003293 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003294
3295 video_stream_encoder_->Stop();
3296}
3297
3298TEST_F(BalancedDegradationTest, AdaptDownReturnsTrueIfFpsDiffGeThreshold) {
3299 test::ScopedFieldTrials field_trials(
3300 "WebRTC-Video-BalancedDegradationSettings/"
3301 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3302 SetupTest();
3303
3304 // Force input frame rate.
3305 const int kInputFps = 25;
3306 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3307 stats.input_frame_rate = kInputFps;
3308 stats_proxy_->SetMockStats(stats);
3309
Åsa Persson45b176f2019-09-30 11:19:05 +02003310 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003311 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003312
3313 // Trigger adapt down, expect scaled down framerate (640x360@24fps).
Henrik Boström91aa7322020-04-28 12:24:33 +02003314 // Fps diff (input-requested:1) == threshold, expect adapting down to clear QP
3315 // samples.
3316 EXPECT_TRUE(
3317 video_stream_encoder_
3318 ->TriggerQualityScalerHighQpAndReturnIfQpSamplesShouldBeCleared());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003319 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003320
3321 video_stream_encoder_->Stop();
3322}
3323
3324TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3325 test::ScopedFieldTrials field_trials(
3326 "WebRTC-Video-BalancedDegradationSettings/"
3327 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3328 SetupTest();
3329
3330 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3331
Åsa Persson45b176f2019-09-30 11:19:05 +02003332 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003333 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003334
3335 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3336 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003337 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003338
3339 video_stream_encoder_->Stop();
3340}
3341
Åsa Perssonccfb3402019-09-25 15:13:04 +02003342TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003343 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003344 "WebRTC-Video-BalancedDegradationSettings/"
3345 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003346 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003347
Åsa Persson1b247f12019-08-14 17:26:39 +02003348 const int kMinBitrateBps = 425000;
3349 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003350 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003351
Åsa Persson45b176f2019-09-30 11:19:05 +02003352 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003353 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003354 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3355
3356 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3357 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003358 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003359 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003360 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3361
3362 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3363 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003364 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003365 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003366 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3367
Åsa Persson30ab0152019-08-27 12:22:33 +02003368 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3369 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003370 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003371 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003372 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003373 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3374
3375 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003376 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003377 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003378 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003379
Åsa Persson30ab0152019-08-27 12:22:33 +02003380 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003381 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003382 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003383 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003384 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003385 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3386
3387 video_stream_encoder_->Stop();
3388}
3389
Åsa Perssonccfb3402019-09-25 15:13:04 +02003390TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003391 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3392 test::ScopedFieldTrials field_trials(
3393 "WebRTC-Video-BalancedDegradationSettings/"
3394 "pixels:57600|129600|230400,fps:7|24|24/");
3395 SetupTest();
3396 OnBitrateUpdated(kLowTargetBitrateBps);
3397
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003398 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003399
3400 // Insert frame, expect scaled down:
3401 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3402 InsertFrame();
3403 EXPECT_FALSE(WaitForFrame(1000));
3404 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3405 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3406
3407 // Insert frame, expect scaled down:
3408 // resolution (320x180@24fps).
3409 InsertFrame();
3410 EXPECT_FALSE(WaitForFrame(1000));
3411 EXPECT_LT(source_.sink_wants().max_pixel_count,
3412 source_.last_wants().max_pixel_count);
3413 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3414
3415 // Frame should not be dropped (min pixels per frame reached).
3416 InsertFrameAndWaitForEncoded();
3417
3418 video_stream_encoder_->Stop();
3419}
3420
3421TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003422 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003423 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003424 "WebRTC-Video-BalancedDegradationSettings/"
3425 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003426 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003427
Åsa Persson30ab0152019-08-27 12:22:33 +02003428 const int kResolutionMinBitrateBps = 435000;
3429 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003430 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003431
Åsa Persson45b176f2019-09-30 11:19:05 +02003432 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003433 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003434 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
3436 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3437 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003438 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003439 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003440 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3441
3442 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3443 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003444 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003445 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003446 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3447
3448 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3449 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003450 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003451 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003452 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3453
Åsa Persson30ab0152019-08-27 12:22:33 +02003454 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3455 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003456 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003457 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003458 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3459
3460 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3461 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003462 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003463 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3464
3465 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003466 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003467 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003468 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003469 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003470 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3471
3472 video_stream_encoder_->Stop();
3473}
3474
Åsa Perssonccfb3402019-09-25 15:13:04 +02003475TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003476 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003477 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003478 "WebRTC-Video-BalancedDegradationSettings/"
3479 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003480 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003481
Åsa Persson30ab0152019-08-27 12:22:33 +02003482 const int kMinBitrateBps = 425000;
3483 const int kTooLowMinBitrateBps = 424000;
3484 const int kResolutionMinBitrateBps = 435000;
3485 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003486 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003487
Åsa Persson45b176f2019-09-30 11:19:05 +02003488 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003489 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003490 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491
3492 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3493 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003494 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003495 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003496 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
3498 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3499 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003500 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003501 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3505 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003506 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003507 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003508 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3509
3510 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3511 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003512 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003513 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3514
3515 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003516 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003517 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003518 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003519 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003520 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521
3522 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003523 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003524 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003525 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003526 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3527
3528 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003529 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003530 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003531 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003532 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003533 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3534
Åsa Persson1b247f12019-08-14 17:26:39 +02003535 video_stream_encoder_->Stop();
3536}
3537
mflodmancc3d4422017-08-03 08:27:51 -07003538TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003539 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3540 const int kWidth = 1280;
3541 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003543 DataRate::BitsPerSec(kTargetBitrateBps),
3544 DataRate::BitsPerSec(kTargetBitrateBps),
3545 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003546
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003547 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07003548 AdaptingFrameForwarder source;
3549 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003550 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003551 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003552
Åsa Persson8c1bf952018-09-13 10:42:19 +02003553 int64_t timestamp_ms = kFrameIntervalMs;
3554 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003555 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003556 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003557 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3558 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3559 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3560 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3561
3562 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07003563 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003564 timestamp_ms += kFrameIntervalMs;
3565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3566 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003567 EXPECT_THAT(source.sink_wants(),
3568 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003569 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3571 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3572 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573
3574 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07003575 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003579 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003580 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3582 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3583 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584
Jonathan Yubc771b72017-12-08 17:04:29 -08003585 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07003586 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003587 timestamp_ms += kFrameIntervalMs;
3588 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3589 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003590 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003591 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3592 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003593 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3595
Jonathan Yubc771b72017-12-08 17:04:29 -08003596 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07003597 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003601 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003602 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07003603 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3606 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3607
Jonathan Yubc771b72017-12-08 17:04:29 -08003608 // Trigger quality adapt down, expect no change (min resolution reached).
3609 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003610 timestamp_ms += kFrameIntervalMs;
3611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3612 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003613 EXPECT_THAT(source.sink_wants(), FpsMax());
3614 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08003615 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3617 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3618 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3619
Evan Shrubsole64469032020-06-11 10:45:29 +02003620 // Trigger quality adapt up, expect upscaled resolution (480x270).
3621 video_stream_encoder_->TriggerQualityHigh();
3622 timestamp_ms += kFrameIntervalMs;
3623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3624 WaitForEncodedFrame(timestamp_ms);
3625 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
3626 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3628 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3630
3631 // Trigger quality and cpu adapt up since both are most limited, expect
3632 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02003633 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003634 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003635 timestamp_ms += kFrameIntervalMs;
3636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3637 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003638 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08003639 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3641 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003642 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08003643
Evan Shrubsole64469032020-06-11 10:45:29 +02003644 // Trigger quality and cpu adapt up since both are most limited, expect
3645 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02003646 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02003647 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003648 timestamp_ms += kFrameIntervalMs;
3649 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3650 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003651 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07003652 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02003653 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07003654 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02003655 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3656 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003657
Evan Shrubsole64469032020-06-11 10:45:29 +02003658 // Trigger cpu adapt up, expect no change since not most limited (960x540).
3659 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02003660 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003661 timestamp_ms += kFrameIntervalMs;
3662 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3663 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003664 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07003665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3666 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003667 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003668 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07003669
3670 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07003671 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003672 timestamp_ms += kFrameIntervalMs;
3673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003674 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003676 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08003679 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02003680 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08003681
mflodmancc3d4422017-08-03 08:27:51 -07003682 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08003683}
3684
mflodmancc3d4422017-08-03 08:27:51 -07003685TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07003686 const int kWidth = 640;
3687 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07003688
Henrik Boström381d1092020-05-12 18:49:07 +02003689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003690 DataRate::BitsPerSec(kTargetBitrateBps),
3691 DataRate::BitsPerSec(kTargetBitrateBps),
3692 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07003693
perkj803d97f2016-11-01 11:45:46 -07003694 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003695 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003696 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07003697 }
3698
mflodmancc3d4422017-08-03 08:27:51 -07003699 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07003700 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07003701 video_source_.IncomingCapturedFrame(CreateFrame(
3702 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003703 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07003704 }
3705
mflodmancc3d4422017-08-03 08:27:51 -07003706 video_stream_encoder_->Stop();
3707 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07003708 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08003709
Ying Wangef3998f2019-12-09 13:06:53 +01003710 EXPECT_METRIC_EQ(
3711 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3712 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07003713 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
3714}
3715
mflodmancc3d4422017-08-03 08:27:51 -07003716TEST_F(VideoStreamEncoderTest,
3717 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003719 DataRate::BitsPerSec(kTargetBitrateBps),
3720 DataRate::BitsPerSec(kTargetBitrateBps),
3721 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07003722 const int kWidth = 640;
3723 const int kHeight = 360;
3724
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003725 video_stream_encoder_->SetSource(&video_source_,
3726 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07003727
3728 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
3729 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003730 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07003731 }
3732
mflodmancc3d4422017-08-03 08:27:51 -07003733 video_stream_encoder_->Stop();
3734 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07003735 stats_proxy_.reset();
3736
3737 EXPECT_EQ(0,
3738 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
3739}
3740
mflodmancc3d4422017-08-03 08:27:51 -07003741TEST_F(VideoStreamEncoderTest, CallsBitrateObserver) {
sprang4847ae62017-06-27 07:06:52 -07003742 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02003743 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
sprang57c2fff2017-01-16 06:24:02 -08003744
3745 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02003746 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01003747 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02003748 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
3749 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08003750
sprang57c2fff2017-01-16 06:24:02 -08003751 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
Niels Möller6bb5ab92019-01-11 11:11:10 +01003752 .Times(1);
Henrik Boström381d1092020-05-12 18:49:07 +02003753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003754 DataRate::BitsPerSec(kLowTargetBitrateBps),
3755 DataRate::BitsPerSec(kLowTargetBitrateBps),
3756 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08003757
sprang57c2fff2017-01-16 06:24:02 -08003758 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003759 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3760 WaitForEncodedFrame(rtc::TimeMillis());
Erik Språng5056af02019-09-02 15:53:11 +02003761 VideoBitrateAllocation bitrate_allocation =
3762 fake_encoder_.GetAndResetLastRateControlSettings()->bitrate;
Erik Språngd7329ca2019-02-21 21:19:53 +01003763 // Check that encoder has been updated too, not just allocation observer.
Erik Språng5056af02019-09-02 15:53:11 +02003764 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
Sebastian Jansson40889f32019-04-17 12:11:20 +02003765 // TODO(srte): The use of millisecs here looks like an error, but the tests
3766 // fails using seconds, this should be investigated.
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003767 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003768
3769 // Not called on second frame.
3770 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3771 .Times(0);
3772 video_source_.IncomingCapturedFrame(
Erik Språngd7329ca2019-02-21 21:19:53 +01003773 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3774 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003775 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08003776
3777 // Called after a process interval.
sprang57c2fff2017-01-16 06:24:02 -08003778 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(expected_bitrate))
3779 .Times(1);
Erik Språngd7329ca2019-02-21 21:19:53 +01003780 const int64_t start_time_ms = rtc::TimeMillis();
3781 while (rtc::TimeMillis() - start_time_ms < kProcessIntervalMs) {
3782 video_source_.IncomingCapturedFrame(
3783 CreateFrame(rtc::TimeMillis(), codec_width_, codec_height_));
3784 WaitForEncodedFrame(rtc::TimeMillis());
Danil Chapovalov0c626af2020-02-10 11:16:00 +01003785 fake_clock_.AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01003786 }
3787
3788 // Since rates are unchanged, encoder should not be reconfigured.
Erik Språng5056af02019-09-02 15:53:11 +02003789 EXPECT_FALSE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
sprang57c2fff2017-01-16 06:24:02 -08003790
mflodmancc3d4422017-08-03 08:27:51 -07003791 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08003792}
3793
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003794TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
3795 // 2 TLs configured, temporal layers supported by encoder.
3796 const int kNumTemporalLayers = 2;
3797 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false);
3798 fake_encoder_.SetTemporalLayersSupported(0, true);
3799
3800 // Bitrate allocated across temporal layers.
3801 const int kTl0Bps = kTargetBitrateBps *
3802 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003803 kNumTemporalLayers, /*temporal_id*/ 0,
3804 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003805 const int kTl1Bps = kTargetBitrateBps *
3806 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003807 kNumTemporalLayers, /*temporal_id*/ 1,
3808 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003809 VideoBitrateAllocation expected_bitrate;
3810 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
3811 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
3812
3813 VerifyAllocatedBitrate(expected_bitrate);
3814 video_stream_encoder_->Stop();
3815}
3816
3817TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
3818 // 2 TLs configured, temporal layers not supported by encoder.
3819 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3820 fake_encoder_.SetTemporalLayersSupported(0, false);
3821
3822 // Temporal layers not supported by the encoder.
3823 // Total bitrate should be at ti:0.
3824 VideoBitrateAllocation expected_bitrate;
3825 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
3826
3827 VerifyAllocatedBitrate(expected_bitrate);
3828 video_stream_encoder_->Stop();
3829}
3830
3831TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
3832 // 2 TLs configured, temporal layers only supported for first stream.
3833 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false);
3834 fake_encoder_.SetTemporalLayersSupported(0, true);
3835 fake_encoder_.SetTemporalLayersSupported(1, false);
3836
3837 const int kS0Bps = 150000;
3838 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003839 kS0Bps *
3840 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3841 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003842 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01003843 kS0Bps *
3844 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
3845 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01003846 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
3847 // Temporal layers not supported by si:1.
3848 VideoBitrateAllocation expected_bitrate;
3849 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
3850 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
3851 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
3852
3853 VerifyAllocatedBitrate(expected_bitrate);
3854 video_stream_encoder_->Stop();
3855}
3856
Niels Möller7dc26b72017-12-06 10:27:48 +01003857TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
3858 const int kFrameWidth = 1280;
3859 const int kFrameHeight = 720;
3860 const int kFramerate = 24;
3861
Henrik Boström381d1092020-05-12 18:49:07 +02003862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003863 DataRate::BitsPerSec(kTargetBitrateBps),
3864 DataRate::BitsPerSec(kTargetBitrateBps),
3865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003866 test::FrameForwarder source;
3867 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003868 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003869
3870 // Insert a single frame, triggering initial configuration.
3871 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3872 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3873
3874 EXPECT_EQ(
3875 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3876 kDefaultFramerate);
3877
3878 // Trigger reconfigure encoder (without resetting the entire instance).
3879 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003880 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003881 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3882 video_encoder_config.number_of_streams = 1;
3883 video_encoder_config.video_stream_factory =
3884 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
3885 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003886 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003887 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3888
3889 // Detector should be updated with fps limit from codec config.
3890 EXPECT_EQ(
3891 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3892 kFramerate);
3893
3894 // Trigger overuse, max framerate should be reduced.
3895 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3896 stats.input_frame_rate = kFramerate;
3897 stats_proxy_->SetMockStats(stats);
3898 video_stream_encoder_->TriggerCpuOveruse();
3899 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3900 int adapted_framerate =
3901 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3902 EXPECT_LT(adapted_framerate, kFramerate);
3903
3904 // Trigger underuse, max framerate should go back to codec configured fps.
3905 // Set extra low fps, to make sure it's actually reset, not just incremented.
3906 stats = stats_proxy_->GetStats();
3907 stats.input_frame_rate = adapted_framerate / 2;
3908 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003909 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003910 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3911 EXPECT_EQ(
3912 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3913 kFramerate);
3914
3915 video_stream_encoder_->Stop();
3916}
3917
3918TEST_F(VideoStreamEncoderTest,
3919 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
3920 const int kFrameWidth = 1280;
3921 const int kFrameHeight = 720;
3922 const int kLowFramerate = 15;
3923 const int kHighFramerate = 25;
3924
Henrik Boström381d1092020-05-12 18:49:07 +02003925 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003926 DataRate::BitsPerSec(kTargetBitrateBps),
3927 DataRate::BitsPerSec(kTargetBitrateBps),
3928 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01003929 test::FrameForwarder source;
3930 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003931 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01003932
3933 // Trigger initial configuration.
3934 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02003935 video_encoder_config.codec_type = kVideoCodecVP8;
Niels Möller7dc26b72017-12-06 10:27:48 +01003936 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
3937 video_encoder_config.number_of_streams = 1;
3938 video_encoder_config.video_stream_factory =
3939 new rtc::RefCountedObject<VideoStreamFactory>(1, kLowFramerate);
3940 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3941 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003942 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003943 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3944
3945 EXPECT_EQ(
3946 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3947 kLowFramerate);
3948
3949 // Trigger overuse, max framerate should be reduced.
3950 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3951 stats.input_frame_rate = kLowFramerate;
3952 stats_proxy_->SetMockStats(stats);
3953 video_stream_encoder_->TriggerCpuOveruse();
3954 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3955 int adapted_framerate =
3956 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
3957 EXPECT_LT(adapted_framerate, kLowFramerate);
3958
3959 // Reconfigure the encoder with a new (higher max framerate), max fps should
3960 // still respect the adaptation.
3961 video_encoder_config.video_stream_factory =
3962 new rtc::RefCountedObject<VideoStreamFactory>(1, kHighFramerate);
3963 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
3964 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003965 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01003966 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3967
3968 EXPECT_EQ(
3969 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3970 adapted_framerate);
3971
3972 // Trigger underuse, max framerate should go back to codec configured fps.
3973 stats = stats_proxy_->GetStats();
3974 stats.input_frame_rate = adapted_framerate;
3975 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02003976 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01003977 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
3978 EXPECT_EQ(
3979 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
3980 kHighFramerate);
3981
3982 video_stream_encoder_->Stop();
3983}
3984
mflodmancc3d4422017-08-03 08:27:51 -07003985TEST_F(VideoStreamEncoderTest,
3986 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07003987 const int kFrameWidth = 1280;
3988 const int kFrameHeight = 720;
3989 const int kFramerate = 24;
3990
Henrik Boström381d1092020-05-12 18:49:07 +02003991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003992 DataRate::BitsPerSec(kTargetBitrateBps),
3993 DataRate::BitsPerSec(kTargetBitrateBps),
3994 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07003995 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003997 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07003998
3999 // Trigger initial configuration.
4000 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004001 video_encoder_config.codec_type = kVideoCodecVP8;
sprangfda496a2017-06-15 04:21:07 -07004002 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4003 video_encoder_config.number_of_streams = 1;
4004 video_encoder_config.video_stream_factory =
4005 new rtc::RefCountedObject<VideoStreamFactory>(1, kFramerate);
4006 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07004007 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004008 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004009 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07004010
Niels Möller7dc26b72017-12-06 10:27:48 +01004011 EXPECT_EQ(
4012 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4013 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004014
4015 // Trigger overuse, max framerate should be reduced.
4016 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4017 stats.input_frame_rate = kFramerate;
4018 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07004019 video_stream_encoder_->TriggerCpuOveruse();
4020 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01004021 int adapted_framerate =
4022 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07004023 EXPECT_LT(adapted_framerate, kFramerate);
4024
4025 // Change degradation preference to not enable framerate scaling. Target
4026 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02004027 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004028 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01004029 EXPECT_EQ(
4030 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4031 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07004032
mflodmancc3d4422017-08-03 08:27:51 -07004033 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07004034}
4035
mflodmancc3d4422017-08-03 08:27:51 -07004036TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004037 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004039 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4040 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4041 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004042 const int kWidth = 640;
4043 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004044
asaperssonfab67072017-04-04 05:51:49 -07004045 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004046
4047 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004048 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004049
4050 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004051 EXPECT_TRUE_WAIT(
4052 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004053
sprangc5d62e22017-04-02 23:53:04 -07004054 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08004055
asaperssonfab67072017-04-04 05:51:49 -07004056 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08004057 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07004058 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08004059
4060 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07004061 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004062
Henrik Boström2671dac2020-05-19 16:29:09 +02004063 EXPECT_TRUE_WAIT(
4064 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08004065
mflodmancc3d4422017-08-03 08:27:51 -07004066 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004067}
4068
mflodmancc3d4422017-08-03 08:27:51 -07004069TEST_F(VideoStreamEncoderTest,
4070 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07004071 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02004072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004073 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4074 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4075 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07004076 const int kWidth = 640;
4077 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08004078
4079 // We expect the n initial frames to get dropped.
4080 int i;
4081 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004082 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004083 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08004084 }
4085 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07004086 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004087 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08004088
4089 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07004090 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08004091
mflodmancc3d4422017-08-03 08:27:51 -07004092 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004093}
4094
mflodmancc3d4422017-08-03 08:27:51 -07004095TEST_F(VideoStreamEncoderTest,
4096 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07004097 const int kWidth = 640;
4098 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02004099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004100 DataRate::BitsPerSec(kLowTargetBitrateBps),
4101 DataRate::BitsPerSec(kLowTargetBitrateBps),
4102 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08004103
4104 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07004105 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004106 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08004107
asaperssonfab67072017-04-04 05:51:49 -07004108 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08004109 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004110 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08004111
mflodmancc3d4422017-08-03 08:27:51 -07004112 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08004113}
4114
mflodmancc3d4422017-08-03 08:27:51 -07004115TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07004116 const int kWidth = 640;
4117 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08004118 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02004119
4120 VideoEncoderConfig video_encoder_config;
4121 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4122 // Make format different, to force recreation of encoder.
4123 video_encoder_config.video_format.parameters["foo"] = "foo";
4124 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004125 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02004126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004127 DataRate::BitsPerSec(kLowTargetBitrateBps),
4128 DataRate::BitsPerSec(kLowTargetBitrateBps),
4129 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07004130
kthelgasonb83797b2017-02-14 11:57:25 -08004131 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004132 video_stream_encoder_->SetSource(&video_source_,
4133 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08004134
asaperssonfab67072017-04-04 05:51:49 -07004135 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08004136 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07004137 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08004138
mflodmancc3d4422017-08-03 08:27:51 -07004139 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08004140 fake_encoder_.SetQualityScaling(true);
4141}
4142
Åsa Persson139f4dc2019-08-02 09:29:58 +02004143TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
4144 webrtc::test::ScopedFieldTrials field_trials(
4145 "WebRTC-Video-QualityScalerSettings/"
4146 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4147 // Reset encoder for field trials to take effect.
4148 ConfigureEncoder(video_encoder_config_.Copy());
4149 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
4150 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
4151 const int kWidth = 640;
4152 const int kHeight = 360;
4153
Henrik Boström381d1092020-05-12 18:49:07 +02004154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004155 DataRate::BitsPerSec(kTargetBitrateBps),
4156 DataRate::BitsPerSec(kTargetBitrateBps),
4157 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004158 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
4159 // Frame should not be dropped.
4160 WaitForEncodedFrame(1);
4161
Henrik Boström381d1092020-05-12 18:49:07 +02004162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004163 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4164 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
4165 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004166 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
4167 // Frame should not be dropped.
4168 WaitForEncodedFrame(2);
4169
Henrik Boström381d1092020-05-12 18:49:07 +02004170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004171 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4172 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
4173 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004174 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
4175 // Expect to drop this frame, the wait should time out.
4176 ExpectDroppedFrame();
4177
4178 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02004179 EXPECT_TRUE_WAIT(
4180 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02004181 video_stream_encoder_->Stop();
4182}
4183
Åsa Perssone644a032019-11-08 15:56:00 +01004184TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
4185 webrtc::test::ScopedFieldTrials field_trials(
4186 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
4187
4188 // Reset encoder for field trials to take effect.
4189 VideoEncoderConfig config = video_encoder_config_.Copy();
4190 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02004191 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01004192 ConfigureEncoder(std::move(config));
4193 fake_encoder_.SetQp(kQpLow);
4194
4195 // Enable MAINTAIN_FRAMERATE preference.
4196 AdaptingFrameForwarder source;
4197 source.set_adaptation_enabled(true);
4198 video_stream_encoder_->SetSource(&source,
4199 DegradationPreference::MAINTAIN_FRAMERATE);
4200
4201 // Start at low bitrate.
4202 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02004203 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4204 DataRate::BitsPerSec(kLowBitrateBps),
4205 DataRate::BitsPerSec(kLowBitrateBps),
4206 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004207
4208 // Expect first frame to be dropped and resolution to be limited.
4209 const int kWidth = 1280;
4210 const int kHeight = 720;
4211 const int64_t kFrameIntervalMs = 100;
4212 int64_t timestamp_ms = kFrameIntervalMs;
4213 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4214 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004215 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
4216 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01004217
4218 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02004219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4220 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01004221
4222 // Insert frames and advance |min_duration_ms|.
4223 for (size_t i = 1; i <= 10; i++) {
4224 timestamp_ms += kFrameIntervalMs;
4225 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4226 WaitForEncodedFrame(timestamp_ms);
4227 }
4228 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4229 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
4230
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004231 fake_clock_.AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01004232
4233 // Insert frame should trigger high BW and release quality limitation.
4234 timestamp_ms += kFrameIntervalMs;
4235 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4236 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02004237 // The ramp-up code involves the adaptation queue, give it time to execute.
4238 // TODO(hbos): Can we await an appropriate event instead?
4239 video_stream_encoder_->WaitUntilAdaptationTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004240 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01004241
4242 // Frame should not be adapted.
4243 timestamp_ms += kFrameIntervalMs;
4244 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4245 WaitForEncodedFrame(kWidth, kHeight);
4246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4247
4248 video_stream_encoder_->Stop();
4249}
4250
mflodmancc3d4422017-08-03 08:27:51 -07004251TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004252 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
4253 const int kTooSmallWidth = 10;
4254 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02004255 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004256 DataRate::BitsPerSec(kTargetBitrateBps),
4257 DataRate::BitsPerSec(kTargetBitrateBps),
4258 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004259
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004260 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07004261 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07004262 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004263 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004264 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07004265 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4266
4267 // Trigger adapt down, too small frame, expect no change.
4268 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004269 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004270 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004271 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4273 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4274
mflodmancc3d4422017-08-03 08:27:51 -07004275 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07004276}
4277
mflodmancc3d4422017-08-03 08:27:51 -07004278TEST_F(VideoStreamEncoderTest,
4279 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004280 const int kTooSmallWidth = 10;
4281 const int kTooSmallHeight = 10;
4282 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02004283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004284 DataRate::BitsPerSec(kTargetBitrateBps),
4285 DataRate::BitsPerSec(kTargetBitrateBps),
4286 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004287
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004288 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004289 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004290 video_stream_encoder_->SetSource(&source,
4291 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004292 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07004293 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4295
4296 // Trigger adapt down, expect limited framerate.
4297 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004298 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07004299 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004300 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004301 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4302 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4303 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4304
4305 // Trigger adapt down, too small frame, expect no change.
4306 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07004307 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07004308 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004309 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4312 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
mflodmancc3d4422017-08-03 08:27:51 -07004314 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004315}
4316
mflodmancc3d4422017-08-03 08:27:51 -07004317TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07004318 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02004319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004320 DataRate::BitsPerSec(kTargetBitrateBps),
4321 DataRate::BitsPerSec(kTargetBitrateBps),
4322 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02004323 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07004324 const int kFrameWidth = 1280;
4325 const int kFrameHeight = 720;
4326 video_source_.IncomingCapturedFrame(
4327 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004328 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07004329 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07004330}
4331
sprangb1ca0732017-02-01 08:38:12 -08004332// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07004333TEST_F(VideoStreamEncoderTest,
4334 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02004335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004336 DataRate::BitsPerSec(kTargetBitrateBps),
4337 DataRate::BitsPerSec(kTargetBitrateBps),
4338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08004339
4340 const int kFrameWidth = 1280;
4341 const int kFrameHeight = 720;
4342 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07004343 // requested by
4344 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08004345 video_source_.set_adaptation_enabled(true);
4346
4347 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004348 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004349 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004350
4351 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004352 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08004353 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004354 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004355 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08004356
asaperssonfab67072017-04-04 05:51:49 -07004357 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02004358 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08004359 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02004360 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004361 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08004362
mflodmancc3d4422017-08-03 08:27:51 -07004363 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08004364}
sprangfe627f32017-03-29 08:24:59 -07004365
mflodmancc3d4422017-08-03 08:27:51 -07004366TEST_F(VideoStreamEncoderTest,
4367 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07004368 const int kFrameWidth = 1280;
4369 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07004370
Henrik Boström381d1092020-05-12 18:49:07 +02004371 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004372 DataRate::BitsPerSec(kTargetBitrateBps),
4373 DataRate::BitsPerSec(kTargetBitrateBps),
4374 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004375 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004376 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004377 video_source_.set_adaptation_enabled(true);
4378
sprang4847ae62017-06-27 07:06:52 -07004379 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004380
4381 video_source_.IncomingCapturedFrame(
4382 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004383 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004384
4385 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07004386 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004387
4388 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07004389 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004390 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004391 video_source_.IncomingCapturedFrame(
4392 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004393 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07004394 }
4395
4396 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07004397 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004398 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004399 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004400 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004401 video_source_.IncomingCapturedFrame(
4402 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004403 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004404 ++num_frames_dropped;
4405 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004406 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004407 }
4408 }
4409
sprang4847ae62017-06-27 07:06:52 -07004410 // Add some slack to account for frames dropped by the frame dropper.
4411 const int kErrorMargin = 1;
4412 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004413 kErrorMargin);
4414
4415 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07004416 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07004417 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02004418 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004419 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004420 video_source_.IncomingCapturedFrame(
4421 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004422 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004423 ++num_frames_dropped;
4424 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004425 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004426 }
4427 }
sprang4847ae62017-06-27 07:06:52 -07004428 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07004429 kErrorMargin);
4430
4431 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02004432 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004433 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004434 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004435 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004436 video_source_.IncomingCapturedFrame(
4437 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004438 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004439 ++num_frames_dropped;
4440 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004441 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004442 }
4443 }
sprang4847ae62017-06-27 07:06:52 -07004444 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07004445 kErrorMargin);
4446
4447 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02004448 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07004449 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07004450 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07004451 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07004452 video_source_.IncomingCapturedFrame(
4453 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004454 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07004455 ++num_frames_dropped;
4456 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01004457 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07004458 }
4459 }
4460 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
4461
mflodmancc3d4422017-08-03 08:27:51 -07004462 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004463}
4464
mflodmancc3d4422017-08-03 08:27:51 -07004465TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07004466 const int kFramerateFps = 5;
4467 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07004468 const int kFrameWidth = 1280;
4469 const int kFrameHeight = 720;
4470
sprang4847ae62017-06-27 07:06:52 -07004471 // Reconfigure encoder with two temporal layers and screensharing, which will
4472 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02004473 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07004474
Henrik Boström381d1092020-05-12 18:49:07 +02004475 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004476 DataRate::BitsPerSec(kTargetBitrateBps),
4477 DataRate::BitsPerSec(kTargetBitrateBps),
4478 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07004479 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004480 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07004481 video_source_.set_adaptation_enabled(true);
4482
sprang4847ae62017-06-27 07:06:52 -07004483 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
sprangc5d62e22017-04-02 23:53:04 -07004484
4485 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08004486 rtc::VideoSinkWants last_wants;
4487 do {
4488 last_wants = video_source_.sink_wants();
4489
sprangc5d62e22017-04-02 23:53:04 -07004490 // Insert frames to get a new fps estimate...
4491 for (int j = 0; j < kFramerateFps; ++j) {
4492 video_source_.IncomingCapturedFrame(
4493 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08004494 if (video_source_.last_sent_width()) {
4495 sink_.WaitForEncodedFrame(timestamp_ms);
4496 }
sprangc5d62e22017-04-02 23:53:04 -07004497 timestamp_ms += kFrameIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01004498 fake_clock_.AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07004499 }
4500 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07004501 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08004502 } while (video_source_.sink_wants().max_framerate_fps <
4503 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07004504
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004505 EXPECT_THAT(video_source_.sink_wants(),
4506 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07004507
mflodmancc3d4422017-08-03 08:27:51 -07004508 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07004509}
asaperssonf7e294d2017-06-13 23:25:22 -07004510
mflodmancc3d4422017-08-03 08:27:51 -07004511TEST_F(VideoStreamEncoderTest,
4512 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07004513 const int kWidth = 1280;
4514 const int kHeight = 720;
4515 const int64_t kFrameIntervalMs = 150;
4516 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004518 DataRate::BitsPerSec(kTargetBitrateBps),
4519 DataRate::BitsPerSec(kTargetBitrateBps),
4520 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004521
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004522 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004523 AdaptingFrameForwarder source;
4524 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004525 video_stream_encoder_->SetSource(&source,
4526 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004527 timestamp_ms += kFrameIntervalMs;
4528 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004529 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004530 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004531 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4532 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4533 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4534
4535 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004536 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004537 timestamp_ms += kFrameIntervalMs;
4538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004539 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004540 EXPECT_THAT(source.sink_wants(),
4541 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4543 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4544 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4545
4546 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004547 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004548 timestamp_ms += kFrameIntervalMs;
4549 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004550 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004551 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4554 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4555
4556 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004557 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004558 timestamp_ms += kFrameIntervalMs;
4559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004560 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004561 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4563 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4564 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4565
4566 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004567 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004568 timestamp_ms += kFrameIntervalMs;
4569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004570 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004571 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4573 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4574 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4575
4576 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004577 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004578 timestamp_ms += kFrameIntervalMs;
4579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004580 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004581 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4584 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4585
4586 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004587 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004588 timestamp_ms += kFrameIntervalMs;
4589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004590 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004591 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4594 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4595
4596 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07004597 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004598 timestamp_ms += kFrameIntervalMs;
4599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004600 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004601 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004602 rtc::VideoSinkWants last_wants = source.sink_wants();
4603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4605 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4606
4607 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004608 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004609 timestamp_ms += kFrameIntervalMs;
4610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004611 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004612 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07004613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4615 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4616
Evan Shrubsole64469032020-06-11 10:45:29 +02004617 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004618 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004619 timestamp_ms += kFrameIntervalMs;
4620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004621 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004622 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4624 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4625 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4626
4627 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07004628 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004629 timestamp_ms += kFrameIntervalMs;
4630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004631 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004632 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004633 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4635 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4636
4637 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004638 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004639 timestamp_ms += kFrameIntervalMs;
4640 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004641 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004642 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004643 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4644 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4645 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4646
4647 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004648 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004649 timestamp_ms += kFrameIntervalMs;
4650 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004651 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004652 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004653 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4654 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4655 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4656
4657 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004658 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004659 timestamp_ms += kFrameIntervalMs;
4660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004661 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004662 EXPECT_THAT(source.sink_wants(), FpsMax());
4663 EXPECT_EQ(source.sink_wants().max_pixel_count,
4664 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07004665 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4666 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4667 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4668
4669 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004670 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004671 timestamp_ms += kFrameIntervalMs;
4672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004673 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004674 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004675 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4677 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4678
Åsa Persson30ab0152019-08-27 12:22:33 +02004679 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004680 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004681 timestamp_ms += kFrameIntervalMs;
4682 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004683 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004686 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4688 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4689
4690 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004691 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004692 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004693 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4694
mflodmancc3d4422017-08-03 08:27:51 -07004695 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004696}
4697
mflodmancc3d4422017-08-03 08:27:51 -07004698TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07004699 const int kWidth = 1280;
4700 const int kHeight = 720;
4701 const int64_t kFrameIntervalMs = 150;
4702 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004704 DataRate::BitsPerSec(kTargetBitrateBps),
4705 DataRate::BitsPerSec(kTargetBitrateBps),
4706 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004707
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004708 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004709 AdaptingFrameForwarder source;
4710 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004711 video_stream_encoder_->SetSource(&source,
4712 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004713 timestamp_ms += kFrameIntervalMs;
4714 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004715 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004716 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004717 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4718 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4719 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4721 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4722 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4723
4724 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004725 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004726 timestamp_ms += kFrameIntervalMs;
4727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004728 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004729 EXPECT_THAT(source.sink_wants(),
4730 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4732 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4733 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4734 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4735 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4736 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4737
4738 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004739 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004740 timestamp_ms += kFrameIntervalMs;
4741 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004742 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004743 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004744 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4745 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4746 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4747 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4748 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4749 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4750
4751 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004752 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004753 timestamp_ms += kFrameIntervalMs;
4754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004755 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004756 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02004757 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07004758 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4759 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4760 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4761 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4762 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4763
Evan Shrubsole64469032020-06-11 10:45:29 +02004764 // Trigger cpu adapt up, expect no change since QP is most limited.
4765 {
4766 // Store current sink wants since we expect no change and if there is no
4767 // change then last_wants() is not updated.
4768 auto previous_sink_wants = source.sink_wants();
4769 video_stream_encoder_->TriggerCpuUnderuse();
4770 timestamp_ms += kFrameIntervalMs;
4771 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4772 WaitForEncodedFrame(timestamp_ms);
4773 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4775 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4776 }
4777
4778 // Trigger quality adapt up, expect increased fps (640x360@30fps).
4779 video_stream_encoder_->TriggerQualityHigh();
4780 timestamp_ms += kFrameIntervalMs;
4781 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4782 WaitForEncodedFrame(timestamp_ms);
4783 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
4784 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4785 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4786 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4787 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4788 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4789 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4790
4791 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4792 // expect increased resolution (960x540@30fps).
4793 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004794 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004795 timestamp_ms += kFrameIntervalMs;
4796 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004797 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02004798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004799 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4800 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4801 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4803 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004804 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004805
Evan Shrubsole64469032020-06-11 10:45:29 +02004806 // Trigger quality adapt up and Cpu adapt up since both are most limited,
4807 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004808 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02004809 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004810 timestamp_ms += kFrameIntervalMs;
4811 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004812 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004813 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004815 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4817 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4819 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004820 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004821
4822 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004823 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004824 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004825 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004826 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004827
mflodmancc3d4422017-08-03 08:27:51 -07004828 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004829}
4830
mflodmancc3d4422017-08-03 08:27:51 -07004831TEST_F(VideoStreamEncoderTest,
4832 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07004833 const int kWidth = 640;
4834 const int kHeight = 360;
4835 const int kFpsLimit = 15;
4836 const int64_t kFrameIntervalMs = 150;
4837 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02004838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004839 DataRate::BitsPerSec(kTargetBitrateBps),
4840 DataRate::BitsPerSec(kTargetBitrateBps),
4841 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07004842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004843 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07004844 AdaptingFrameForwarder source;
4845 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004846 video_stream_encoder_->SetSource(&source,
4847 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004848 timestamp_ms += kFrameIntervalMs;
4849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004850 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004851 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4857 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4858
4859 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004860 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07004861 timestamp_ms += kFrameIntervalMs;
4862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004863 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004864 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07004865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4869 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4870 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4871
4872 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07004873 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07004874 timestamp_ms += kFrameIntervalMs;
4875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004876 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004877 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07004880 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4881 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4882 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4884
Evan Shrubsole64469032020-06-11 10:45:29 +02004885 // Trigger cpu adapt up, expect no change because quality is most limited.
4886 {
4887 auto previous_sink_wants = source.sink_wants();
4888 // Store current sink wants since we expect no change ind if there is no
4889 // change then last__wants() is not updated.
4890 video_stream_encoder_->TriggerCpuUnderuse();
4891 timestamp_ms += kFrameIntervalMs;
4892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4893 WaitForEncodedFrame(timestamp_ms);
4894 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
4895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4896 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4897 }
4898
4899 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
4900 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07004901 timestamp_ms += kFrameIntervalMs;
4902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004903 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004904 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07004905 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
4907 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004908 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
4909 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4910 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004911
Evan Shrubsole64469032020-06-11 10:45:29 +02004912 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07004913 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02004914 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07004915 timestamp_ms += kFrameIntervalMs;
4916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004917 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
4921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
4923 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004924 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004925
4926 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07004927 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004928 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004929 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004930 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07004931
mflodmancc3d4422017-08-03 08:27:51 -07004932 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004933}
4934
mflodmancc3d4422017-08-03 08:27:51 -07004935TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07004936 const int kFrameWidth = 1920;
4937 const int kFrameHeight = 1080;
4938 // 3/4 of 1920.
4939 const int kAdaptedFrameWidth = 1440;
4940 // 3/4 of 1080 rounded down to multiple of 4.
4941 const int kAdaptedFrameHeight = 808;
4942 const int kFramerate = 24;
4943
Henrik Boström381d1092020-05-12 18:49:07 +02004944 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004945 DataRate::BitsPerSec(kTargetBitrateBps),
4946 DataRate::BitsPerSec(kTargetBitrateBps),
4947 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07004948 // Trigger reconfigure encoder (without resetting the entire instance).
4949 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02004950 video_encoder_config.codec_type = kVideoCodecVP8;
ilnik6b826ef2017-06-16 06:53:48 -07004951 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
4952 video_encoder_config.number_of_streams = 1;
4953 video_encoder_config.video_stream_factory =
4954 new rtc::RefCountedObject<CroppingVideoStreamFactory>(1, kFramerate);
mflodmancc3d4422017-08-03 08:27:51 -07004955 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004956 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07004957 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07004958
4959 video_source_.set_adaptation_enabled(true);
4960
4961 video_source_.IncomingCapturedFrame(
4962 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004963 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004964
4965 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07004966 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07004967 video_source_.IncomingCapturedFrame(
4968 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07004969 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07004970
mflodmancc3d4422017-08-03 08:27:51 -07004971 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07004972}
4973
mflodmancc3d4422017-08-03 08:27:51 -07004974TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07004975 const int kFrameWidth = 1280;
4976 const int kFrameHeight = 720;
4977 const int kLowFps = 2;
4978 const int kHighFps = 30;
4979
Henrik Boström381d1092020-05-12 18:49:07 +02004980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004981 DataRate::BitsPerSec(kTargetBitrateBps),
4982 DataRate::BitsPerSec(kTargetBitrateBps),
4983 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004984
4985 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
4986 max_framerate_ = kLowFps;
4987
4988 // Insert 2 seconds of 2fps video.
4989 for (int i = 0; i < kLowFps * 2; ++i) {
4990 video_source_.IncomingCapturedFrame(
4991 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
4992 WaitForEncodedFrame(timestamp_ms);
4993 timestamp_ms += 1000 / kLowFps;
4994 }
4995
4996 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02004997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004998 DataRate::BitsPerSec(kTargetBitrateBps),
4999 DataRate::BitsPerSec(kTargetBitrateBps),
5000 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07005001 video_source_.IncomingCapturedFrame(
5002 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5003 WaitForEncodedFrame(timestamp_ms);
5004 timestamp_ms += 1000 / kLowFps;
5005
5006 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
5007
5008 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02005009 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07005010 const int kFrameIntervalMs = 1000 / kHighFps;
5011 max_framerate_ = kHighFps;
5012 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
5013 video_source_.IncomingCapturedFrame(
5014 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5015 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
5016 // be dropped if the encoder hans't been updated with the new higher target
5017 // framerate yet, causing it to overshoot the target bitrate and then
5018 // suffering the wrath of the media optimizer.
5019 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
5020 timestamp_ms += kFrameIntervalMs;
5021 }
5022
5023 // Don expect correct measurement just yet, but it should be higher than
5024 // before.
5025 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
5026
mflodmancc3d4422017-08-03 08:27:51 -07005027 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005028}
5029
mflodmancc3d4422017-08-03 08:27:51 -07005030TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07005031 const int kFrameWidth = 1280;
5032 const int kFrameHeight = 720;
5033 const int kTargetBitrateBps = 1000000;
5034
5035 MockBitrateObserver bitrate_observer;
Niels Möller0327c2d2018-05-21 14:09:31 +02005036 video_stream_encoder_->SetBitrateAllocationObserver(&bitrate_observer);
Henrik Boström381d1092020-05-12 18:49:07 +02005037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005038 DataRate::BitsPerSec(kTargetBitrateBps),
5039 DataRate::BitsPerSec(kTargetBitrateBps),
5040 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005041 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07005042
5043 // Insert a first video frame, causes another bitrate update.
5044 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5045 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(1);
5046 video_source_.IncomingCapturedFrame(
5047 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5048 WaitForEncodedFrame(timestamp_ms);
5049
5050 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02005051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5052 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
5053 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07005054
5055 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02005056 timestamp_ms += kProcessIntervalMs;
Danil Chapovalov0c626af2020-02-10 11:16:00 +01005057 fake_clock_.AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07005058
5059 // Bitrate observer should not be called.
5060 EXPECT_CALL(bitrate_observer, OnBitrateAllocationUpdated(_)).Times(0);
5061 video_source_.IncomingCapturedFrame(
5062 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5063 ExpectDroppedFrame();
5064
mflodmancc3d4422017-08-03 08:27:51 -07005065 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07005066}
ilnik6b826ef2017-06-16 06:53:48 -07005067
Niels Möller4db138e2018-04-19 09:04:13 +02005068TEST_F(VideoStreamEncoderTest,
5069 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
5070 const int kFrameWidth = 1280;
5071 const int kFrameHeight = 720;
5072 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02005073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005074 DataRate::BitsPerSec(kTargetBitrateBps),
5075 DataRate::BitsPerSec(kTargetBitrateBps),
5076 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005077 video_source_.IncomingCapturedFrame(
5078 CreateFrame(1, kFrameWidth, kFrameHeight));
5079 WaitForEncodedFrame(1);
5080 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5081 .low_encode_usage_threshold_percent,
5082 default_options.low_encode_usage_threshold_percent);
5083 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5084 .high_encode_usage_threshold_percent,
5085 default_options.high_encode_usage_threshold_percent);
5086 video_stream_encoder_->Stop();
5087}
5088
5089TEST_F(VideoStreamEncoderTest,
5090 HigherCpuAdaptationThresholdsForHardwareEncoder) {
5091 const int kFrameWidth = 1280;
5092 const int kFrameHeight = 720;
5093 CpuOveruseOptions hardware_options;
5094 hardware_options.low_encode_usage_threshold_percent = 150;
5095 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01005096 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02005097
Henrik Boström381d1092020-05-12 18:49:07 +02005098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005099 DataRate::BitsPerSec(kTargetBitrateBps),
5100 DataRate::BitsPerSec(kTargetBitrateBps),
5101 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02005102 video_source_.IncomingCapturedFrame(
5103 CreateFrame(1, kFrameWidth, kFrameHeight));
5104 WaitForEncodedFrame(1);
5105 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5106 .low_encode_usage_threshold_percent,
5107 hardware_options.low_encode_usage_threshold_percent);
5108 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
5109 .high_encode_usage_threshold_percent,
5110 hardware_options.high_encode_usage_threshold_percent);
5111 video_stream_encoder_->Stop();
5112}
5113
Niels Möller6bb5ab92019-01-11 11:11:10 +01005114TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
5115 const int kFrameWidth = 320;
5116 const int kFrameHeight = 240;
5117 const int kFps = 30;
5118 const int kTargetBitrateBps = 120000;
5119 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
5120
Henrik Boström381d1092020-05-12 18:49:07 +02005121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005122 DataRate::BitsPerSec(kTargetBitrateBps),
5123 DataRate::BitsPerSec(kTargetBitrateBps),
5124 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005125
5126 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5127 max_framerate_ = kFps;
5128
5129 // Insert 3 seconds of video, verify number of drops with normal bitrate.
5130 fake_encoder_.SimulateOvershoot(1.0);
5131 int num_dropped = 0;
5132 for (int i = 0; i < kNumFramesInRun; ++i) {
5133 video_source_.IncomingCapturedFrame(
5134 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5135 // Wait up to two frame durations for a frame to arrive.
5136 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5137 ++num_dropped;
5138 }
5139 timestamp_ms += 1000 / kFps;
5140 }
5141
Erik Språnga8d48ab2019-02-08 14:17:40 +01005142 // Framerate should be measured to be near the expected target rate.
5143 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5144
5145 // Frame drops should be within 5% of expected 0%.
5146 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005147
5148 // Make encoder produce frames at double the expected bitrate during 3 seconds
5149 // of video, verify number of drops. Rate needs to be slightly changed in
5150 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01005151 double overshoot_factor = 2.0;
5152 if (RateControlSettings::ParseFromFieldTrials().UseEncoderBitrateAdjuster()) {
5153 // With bitrate adjuster, when need to overshoot even more to trigger
5154 // frame dropping.
5155 overshoot_factor *= 2;
5156 }
5157 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02005158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005159 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5160 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
5161 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005162 num_dropped = 0;
5163 for (int i = 0; i < kNumFramesInRun; ++i) {
5164 video_source_.IncomingCapturedFrame(
5165 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5166 // Wait up to two frame durations for a frame to arrive.
5167 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
5168 ++num_dropped;
5169 }
5170 timestamp_ms += 1000 / kFps;
5171 }
5172
Henrik Boström381d1092020-05-12 18:49:07 +02005173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005174 DataRate::BitsPerSec(kTargetBitrateBps),
5175 DataRate::BitsPerSec(kTargetBitrateBps),
5176 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01005177
5178 // Target framerate should be still be near the expected target, despite
5179 // the frame drops.
5180 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
5181
5182 // Frame drops should be within 5% of expected 50%.
5183 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005184
5185 video_stream_encoder_->Stop();
5186}
5187
5188TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
5189 const int kFrameWidth = 320;
5190 const int kFrameHeight = 240;
5191 const int kActualInputFps = 24;
5192 const int kTargetBitrateBps = 120000;
5193
5194 ASSERT_GT(max_framerate_, kActualInputFps);
5195
5196 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5197 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02005198 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005199 DataRate::BitsPerSec(kTargetBitrateBps),
5200 DataRate::BitsPerSec(kTargetBitrateBps),
5201 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01005202
5203 // Insert 3 seconds of video, with an input fps lower than configured max.
5204 for (int i = 0; i < kActualInputFps * 3; ++i) {
5205 video_source_.IncomingCapturedFrame(
5206 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
5207 // Wait up to two frame durations for a frame to arrive.
5208 WaitForEncodedFrame(timestamp_ms);
5209 timestamp_ms += 1000 / kActualInputFps;
5210 }
5211
5212 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
5213
5214 video_stream_encoder_->Stop();
5215}
5216
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005217TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
5218 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02005219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005220 DataRate::BitsPerSec(kTargetBitrateBps),
5221 DataRate::BitsPerSec(kTargetBitrateBps),
5222 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01005223
5224 fake_encoder_.BlockNextEncode();
5225 video_source_.IncomingCapturedFrame(
5226 CreateFrameWithUpdatedPixel(1, nullptr, 0));
5227 WaitForEncodedFrame(1);
5228 // On the very first frame full update should be forced.
5229 rect = fake_encoder_.GetLastUpdateRect();
5230 EXPECT_EQ(rect.offset_x, 0);
5231 EXPECT_EQ(rect.offset_y, 0);
5232 EXPECT_EQ(rect.height, codec_height_);
5233 EXPECT_EQ(rect.width, codec_width_);
5234 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
5235 // call to ContinueEncode.
5236 video_source_.IncomingCapturedFrame(
5237 CreateFrameWithUpdatedPixel(2, nullptr, 1));
5238 ExpectDroppedFrame();
5239 video_source_.IncomingCapturedFrame(
5240 CreateFrameWithUpdatedPixel(3, nullptr, 10));
5241 ExpectDroppedFrame();
5242 fake_encoder_.ContinueEncode();
5243 WaitForEncodedFrame(3);
5244 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
5245 rect = fake_encoder_.GetLastUpdateRect();
5246 EXPECT_EQ(rect.offset_x, 1);
5247 EXPECT_EQ(rect.offset_y, 0);
5248 EXPECT_EQ(rect.width, 10);
5249 EXPECT_EQ(rect.height, 1);
5250
5251 video_source_.IncomingCapturedFrame(
5252 CreateFrameWithUpdatedPixel(4, nullptr, 0));
5253 WaitForEncodedFrame(4);
5254 // Previous frame was encoded, so no accumulation should happen.
5255 rect = fake_encoder_.GetLastUpdateRect();
5256 EXPECT_EQ(rect.offset_x, 0);
5257 EXPECT_EQ(rect.offset_y, 0);
5258 EXPECT_EQ(rect.width, 1);
5259 EXPECT_EQ(rect.height, 1);
5260
5261 video_stream_encoder_->Stop();
5262}
5263
Erik Språngd7329ca2019-02-21 21:19:53 +01005264TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02005265 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005266 DataRate::BitsPerSec(kTargetBitrateBps),
5267 DataRate::BitsPerSec(kTargetBitrateBps),
5268 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005269
5270 // First frame is always keyframe.
5271 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5272 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01005273 EXPECT_THAT(
5274 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005275 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005276
5277 // Insert delta frame.
5278 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5279 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01005280 EXPECT_THAT(
5281 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005282 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005283
5284 // Request next frame be a key-frame.
5285 video_stream_encoder_->SendKeyFrame();
5286 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5287 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01005288 EXPECT_THAT(
5289 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005290 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005291
5292 video_stream_encoder_->Stop();
5293}
5294
5295TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
5296 // Setup simulcast with three streams.
5297 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005298 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005299 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5300 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5301 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005302 // Wait for all three layers before triggering event.
5303 sink_.SetNumExpectedLayers(3);
5304
5305 // First frame is always keyframe.
5306 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
5307 WaitForEncodedFrame(1);
5308 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005309 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
5310 VideoFrameType::kVideoFrameKey,
5311 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005312
5313 // Insert delta frame.
5314 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
5315 WaitForEncodedFrame(2);
5316 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005317 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
5318 VideoFrameType::kVideoFrameDelta,
5319 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005320
5321 // Request next frame be a key-frame.
5322 // Only first stream is configured to produce key-frame.
5323 video_stream_encoder_->SendKeyFrame();
5324 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
5325 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02005326
5327 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
5328 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01005329 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005330 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02005331 VideoFrameType::kVideoFrameKey,
5332 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005333
5334 video_stream_encoder_->Stop();
5335}
5336
5337TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
5338 // Configure internal source factory and setup test again.
5339 encoder_factory_.SetHasInternalSource(true);
5340 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005341 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005342 DataRate::BitsPerSec(kTargetBitrateBps),
5343 DataRate::BitsPerSec(kTargetBitrateBps),
5344 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01005345
5346 // Call encoder directly, simulating internal source where encoded frame
5347 // callback in VideoStreamEncoder is called despite no OnFrame().
5348 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
5349 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005350 EXPECT_THAT(
5351 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005352 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005353
Niels Möller8f7ce222019-03-21 15:43:58 +01005354 const std::vector<VideoFrameType> kDeltaFrame = {
5355 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01005356 // Need to set timestamp manually since manually for injected frame.
5357 VideoFrame frame = CreateFrame(101, nullptr);
5358 frame.set_timestamp(101);
5359 fake_encoder_.InjectFrame(frame, false);
5360 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005361 EXPECT_THAT(
5362 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005363 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005364
5365 // Request key-frame. The forces a dummy frame down into the encoder.
5366 fake_encoder_.ExpectNullFrame();
5367 video_stream_encoder_->SendKeyFrame();
5368 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01005369 EXPECT_THAT(
5370 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02005371 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01005372
5373 video_stream_encoder_->Stop();
5374}
Erik Språngb7cb7b52019-02-26 15:52:33 +01005375
5376TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
5377 // Configure internal source factory and setup test again.
5378 encoder_factory_.SetHasInternalSource(true);
5379 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02005380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005381 DataRate::BitsPerSec(kTargetBitrateBps),
5382 DataRate::BitsPerSec(kTargetBitrateBps),
5383 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01005384
5385 int64_t timestamp = 1;
5386 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02005387 image.SetEncodedData(
5388 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01005389 image.capture_time_ms_ = ++timestamp;
5390 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
5391 const int64_t kEncodeFinishDelayMs = 10;
5392 image.timing_.encode_start_ms = timestamp;
5393 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
5394 fake_encoder_.InjectEncodedImage(image);
5395 // Wait for frame without incrementing clock.
5396 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5397 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
5398 // capture timestamp should be kEncodeFinishDelayMs in the past.
5399 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
5400 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec -
5401 kEncodeFinishDelayMs);
5402
5403 video_stream_encoder_->Stop();
5404}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02005405
5406TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
5407 // Configure internal source factory and setup test again.
5408 encoder_factory_.SetHasInternalSource(true);
5409 ResetEncoder("H264", 1, 1, 1, false);
5410
5411 EncodedImage image(optimal_sps, sizeof(optimal_sps), sizeof(optimal_sps));
5412 image._frameType = VideoFrameType::kVideoFrameKey;
5413
5414 CodecSpecificInfo codec_specific_info;
5415 codec_specific_info.codecType = kVideoCodecH264;
5416
5417 RTPFragmentationHeader fragmentation;
5418 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5419 fragmentation.fragmentationOffset[0] = 4;
5420 fragmentation.fragmentationLength[0] = sizeof(optimal_sps) - 4;
5421
5422 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5423 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5424
5425 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5426 testing::ElementsAreArray(optimal_sps));
5427 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5428 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5429 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5430 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5431
5432 video_stream_encoder_->Stop();
5433}
5434
5435TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
5436 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
5437 0x00, 0x00, 0x03, 0x03, 0xF4,
5438 0x05, 0x03, 0xC7, 0xC0};
5439
5440 // Configure internal source factory and setup test again.
5441 encoder_factory_.SetHasInternalSource(true);
5442 ResetEncoder("H264", 1, 1, 1, false);
5443
5444 EncodedImage image(original_sps, sizeof(original_sps), sizeof(original_sps));
5445 image._frameType = VideoFrameType::kVideoFrameKey;
5446
5447 CodecSpecificInfo codec_specific_info;
5448 codec_specific_info.codecType = kVideoCodecH264;
5449
5450 RTPFragmentationHeader fragmentation;
5451 fragmentation.VerifyAndAllocateFragmentationHeader(1);
5452 fragmentation.fragmentationOffset[0] = 4;
5453 fragmentation.fragmentationLength[0] = sizeof(original_sps) - 4;
5454
5455 fake_encoder_.InjectEncodedImage(image, &codec_specific_info, &fragmentation);
5456 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
5457
5458 EXPECT_THAT(sink_.GetLastEncodedImageData(),
5459 testing::ElementsAreArray(optimal_sps));
5460 RTPFragmentationHeader last_fragmentation = sink_.GetLastFragmentation();
5461 ASSERT_THAT(last_fragmentation.fragmentationVectorSize, 1U);
5462 EXPECT_EQ(last_fragmentation.fragmentationOffset[0], 4U);
5463 EXPECT_EQ(last_fragmentation.fragmentationLength[0], sizeof(optimal_sps) - 4);
5464
5465 video_stream_encoder_->Stop();
5466}
5467
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005468TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
5469 const int kFrameWidth = 1280;
5470 const int kFrameHeight = 720;
5471 const int kTargetBitrateBps = 300000; // To low for HD resolution.
5472
Henrik Boström381d1092020-05-12 18:49:07 +02005473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005474 DataRate::BitsPerSec(kTargetBitrateBps),
5475 DataRate::BitsPerSec(kTargetBitrateBps),
5476 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02005477 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5478
5479 // Insert a first video frame. It should be dropped because of downscale in
5480 // resolution.
5481 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5482 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5483 frame.set_rotation(kVideoRotation_270);
5484 video_source_.IncomingCapturedFrame(frame);
5485
5486 ExpectDroppedFrame();
5487
5488 // Second frame is downscaled.
5489 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5490 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5491 frame.set_rotation(kVideoRotation_90);
5492 video_source_.IncomingCapturedFrame(frame);
5493
5494 WaitForEncodedFrame(timestamp_ms);
5495 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
5496
5497 // Insert another frame, also downscaled.
5498 timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5499 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
5500 frame.set_rotation(kVideoRotation_180);
5501 video_source_.IncomingCapturedFrame(frame);
5502
5503 WaitForEncodedFrame(timestamp_ms);
5504 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
5505
5506 video_stream_encoder_->Stop();
5507}
5508
Erik Språng5056af02019-09-02 15:53:11 +02005509TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
5510 const int kFrameWidth = 320;
5511 const int kFrameHeight = 180;
5512
5513 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02005514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005515 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
5516 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
5517 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02005518 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005519 /*rtt_ms=*/0,
5520 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005521
5522 // Insert a first video frame so that encoder gets configured.
5523 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5524 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5525 frame.set_rotation(kVideoRotation_270);
5526 video_source_.IncomingCapturedFrame(frame);
5527 WaitForEncodedFrame(timestamp_ms);
5528
5529 // Set a target rate below the minimum allowed by the codec settings.
5530 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005531 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
5532 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02005533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02005534 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02005535 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02005536 /*link_allocation=*/target_rate,
5537 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005538 /*rtt_ms=*/0,
5539 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02005540 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5541
5542 // Target bitrate and bandwidth allocation should both be capped at min_rate.
5543 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5544 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005545 DataRate allocation_sum =
5546 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02005547 EXPECT_EQ(min_rate, allocation_sum);
5548 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
5549
5550 video_stream_encoder_->Stop();
5551}
5552
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005553TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02005554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005555 DataRate::BitsPerSec(kTargetBitrateBps),
5556 DataRate::BitsPerSec(kTargetBitrateBps),
5557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005558 // Capture a frame and wait for it to synchronize with the encoder thread.
5559 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5560 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5561 WaitForEncodedFrame(1);
5562
5563 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
5564 ASSERT_TRUE(prev_rate_settings.has_value());
5565 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
5566 kDefaultFramerate);
5567
5568 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
5569 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
5570 timestamp_ms += 1000 / kDefaultFramerate;
5571 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5572 WaitForEncodedFrame(timestamp_ms);
5573 }
5574 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
5575 kDefaultFramerate);
5576 // Capture larger frame to trigger a reconfigure.
5577 codec_height_ *= 2;
5578 codec_width_ *= 2;
5579 timestamp_ms += 1000 / kDefaultFramerate;
5580 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
5581 WaitForEncodedFrame(timestamp_ms);
5582
5583 EXPECT_EQ(2, sink_.number_of_reconfigurations());
5584 auto current_rate_settings =
5585 fake_encoder_.GetAndResetLastRateControlSettings();
5586 // Ensure we have actually reconfigured twice
5587 // The rate settings should have been set again even though
5588 // they haven't changed.
5589 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005590 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02005591
5592 video_stream_encoder_->Stop();
5593}
5594
philipeld9cc8c02019-09-16 14:53:40 +02005595struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02005596 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
5597 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
5598 MOCK_METHOD(void,
5599 RequestEncoderSwitch,
5600 (const webrtc::SdpVideoFormat& format),
5601 (override));
philipeld9cc8c02019-09-16 14:53:40 +02005602};
5603
5604TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
5605 constexpr int kDontCare = 100;
5606
5607 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5608 video_send_config_.encoder_settings.encoder_switch_request_callback =
5609 &switch_callback;
5610 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5611 encoder_config.codec_type = kVideoCodecVP8;
5612 webrtc::test::ScopedFieldTrials field_trial(
5613 "WebRTC-NetworkCondition-EncoderSwitch/"
5614 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5615 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5616
5617 // Reset encoder for new configuration to take effect.
5618 ConfigureEncoder(std::move(encoder_config));
5619
5620 // Send one frame to trigger ReconfigureEncoder.
5621 video_source_.IncomingCapturedFrame(
5622 CreateFrame(kDontCare, kDontCare, kDontCare));
5623
5624 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005625 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5626 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005627 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005628 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005629
Henrik Boström381d1092020-05-12 18:49:07 +02005630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005631 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5632 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5633 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02005634 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005635 /*rtt_ms=*/0,
5636 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005637
5638 video_stream_encoder_->Stop();
5639}
5640
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005641TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
5642 constexpr int kDontCare = 100;
5643
5644 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5645 video_send_config_.encoder_settings.encoder_switch_request_callback =
5646 &switch_callback;
5647 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5648 encoder_config.codec_type = kVideoCodecVP8;
5649 webrtc::test::ScopedFieldTrials field_trial(
5650 "WebRTC-NetworkCondition-EncoderSwitch/"
5651 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
5652 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5653
5654 // Reset encoder for new configuration to take effect.
5655 ConfigureEncoder(std::move(encoder_config));
5656
5657 // Send one frame to trigger ReconfigureEncoder.
5658 video_source_.IncomingCapturedFrame(
5659 CreateFrame(kDontCare, kDontCare, kDontCare));
5660
5661 using Config = EncoderSwitchRequestCallback::Config;
5662 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
5663 .Times(0);
5664
Henrik Boström381d1092020-05-12 18:49:07 +02005665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01005666 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
5667 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
5668 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
5669 /*fraction_lost=*/0,
5670 /*rtt_ms=*/0,
5671 /*cwnd_reduce_ratio=*/0);
5672
5673 video_stream_encoder_->Stop();
5674}
5675
philipeld9cc8c02019-09-16 14:53:40 +02005676TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
5677 constexpr int kSufficientBitrateToNotDrop = 1000;
5678 constexpr int kHighRes = 500;
5679 constexpr int kLowRes = 100;
5680
5681 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5682 video_send_config_.encoder_settings.encoder_switch_request_callback =
5683 &switch_callback;
5684 webrtc::test::ScopedFieldTrials field_trial(
5685 "WebRTC-NetworkCondition-EncoderSwitch/"
5686 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
5687 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
5688 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
5689 encoder_config.codec_type = kVideoCodecH264;
5690
5691 // Reset encoder for new configuration to take effect.
5692 ConfigureEncoder(std::move(encoder_config));
5693
5694 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5695 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5696 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005698 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5699 /*stable_target_bitrate=*/
5700 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5701 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02005702 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005703 /*rtt_ms=*/0,
5704 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02005705
5706 // Send one frame to trigger ReconfigureEncoder.
5707 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
5708 WaitForEncodedFrame(1);
5709
5710 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01005711 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
5712 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02005713 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01005714 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02005715
5716 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
5717 WaitForEncodedFrame(2);
5718
5719 video_stream_encoder_->Stop();
5720}
5721
philipel9b058032020-02-10 11:30:00 +01005722TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
5723 constexpr int kDontCare = 100;
5724 StrictMock<MockEncoderSelector> encoder_selector;
5725 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5726 &fake_encoder_, &encoder_selector);
5727 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5728
5729 // Reset encoder for new configuration to take effect.
5730 ConfigureEncoder(video_encoder_config_.Copy());
5731
5732 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
5733
5734 video_source_.IncomingCapturedFrame(
5735 CreateFrame(kDontCare, kDontCare, kDontCare));
5736 video_stream_encoder_->Stop();
5737
5738 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5739 // to it's factory, so in order for the encoder instance in the
5740 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5741 // reset the |video_stream_encoder_| here.
5742 video_stream_encoder_.reset();
5743}
5744
5745TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
5746 constexpr int kDontCare = 100;
5747
5748 NiceMock<MockEncoderSelector> encoder_selector;
5749 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5750 video_send_config_.encoder_settings.encoder_switch_request_callback =
5751 &switch_callback;
5752 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5753 &fake_encoder_, &encoder_selector);
5754 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5755
5756 // Reset encoder for new configuration to take effect.
5757 ConfigureEncoder(video_encoder_config_.Copy());
5758
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01005759 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01005760 .WillByDefault(Return(SdpVideoFormat("AV1")));
5761 EXPECT_CALL(switch_callback,
5762 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
5763 Field(&SdpVideoFormat::name, "AV1"))));
5764
Henrik Boström381d1092020-05-12 18:49:07 +02005765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005766 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
5767 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
5768 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01005769 /*fraction_lost=*/0,
5770 /*rtt_ms=*/0,
5771 /*cwnd_reduce_ratio=*/0);
5772
5773 video_stream_encoder_->Stop();
5774}
5775
5776TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
5777 constexpr int kSufficientBitrateToNotDrop = 1000;
5778 constexpr int kDontCare = 100;
5779
5780 NiceMock<MockVideoEncoder> video_encoder;
5781 NiceMock<MockEncoderSelector> encoder_selector;
5782 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
5783 video_send_config_.encoder_settings.encoder_switch_request_callback =
5784 &switch_callback;
5785 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
5786 &video_encoder, &encoder_selector);
5787 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
5788
5789 // Reset encoder for new configuration to take effect.
5790 ConfigureEncoder(video_encoder_config_.Copy());
5791
5792 // The VideoStreamEncoder needs some bitrate before it can start encoding,
5793 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
5794 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02005795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005796 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5797 /*stable_target_bitrate=*/
5798 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
5799 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01005800 /*fraction_lost=*/0,
5801 /*rtt_ms=*/0,
5802 /*cwnd_reduce_ratio=*/0);
5803
5804 ON_CALL(video_encoder, Encode(_, _))
5805 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
5806 ON_CALL(encoder_selector, OnEncoderBroken())
5807 .WillByDefault(Return(SdpVideoFormat("AV2")));
5808
5809 rtc::Event encode_attempted;
5810 EXPECT_CALL(switch_callback,
5811 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
5812 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
5813 EXPECT_EQ(format.name, "AV2");
5814 encode_attempted.Set();
5815 });
5816
5817 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
5818 encode_attempted.Wait(3000);
5819
5820 video_stream_encoder_->Stop();
5821
5822 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
5823 // to it's factory, so in order for the encoder instance in the
5824 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
5825 // reset the |video_stream_encoder_| here.
5826 video_stream_encoder_.reset();
5827}
5828
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005829TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005830 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005831 const int kFrameWidth = 320;
5832 const int kFrameHeight = 180;
5833
5834 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005835 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005837 /*target_bitrate=*/rate,
5838 /*stable_target_bitrate=*/rate,
5839 /*link_allocation=*/rate,
5840 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005841 /*rtt_ms=*/0,
5842 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005843
5844 // Insert a first video frame so that encoder gets configured.
5845 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5846 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5847 frame.set_rotation(kVideoRotation_270);
5848 video_source_.IncomingCapturedFrame(frame);
5849 WaitForEncodedFrame(timestamp_ms);
5850 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5851
5852 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005853 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005855 /*target_bitrate=*/new_stable_rate,
5856 /*stable_target_bitrate=*/new_stable_rate,
5857 /*link_allocation=*/rate,
5858 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005859 /*rtt_ms=*/0,
5860 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005861 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5862 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
5863 video_stream_encoder_->Stop();
5864}
5865
5866TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01005867 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005868 const int kFrameWidth = 320;
5869 const int kFrameHeight = 180;
5870
5871 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005872 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02005873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005874 /*target_bitrate=*/rate,
5875 /*stable_target_bitrate=*/rate,
5876 /*link_allocation=*/rate,
5877 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005878 /*rtt_ms=*/0,
5879 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005880
5881 // Insert a first video frame so that encoder gets configured.
5882 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5883 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
5884 frame.set_rotation(kVideoRotation_270);
5885 video_source_.IncomingCapturedFrame(frame);
5886 WaitForEncodedFrame(timestamp_ms);
5887 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5888
5889 // Set a higher target rate without changing the link_allocation. Should not
5890 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005891 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02005892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005893 /*target_bitrate=*/rate,
5894 /*stable_target_bitrate=*/new_stable_rate,
5895 /*link_allocation=*/rate,
5896 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01005897 /*rtt_ms=*/0,
5898 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02005899 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5900 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
5901 video_stream_encoder_->Stop();
5902}
5903
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005904TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
5905 test::ScopedFieldTrials field_trials(
5906 "WebRTC-AutomaticAnimationDetectionScreenshare/"
5907 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
5908 const int kFramerateFps = 30;
5909 const int kWidth = 1920;
5910 const int kHeight = 1080;
5911 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
5912 // Works on screenshare mode.
5913 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
5914 // We rely on the automatic resolution adaptation, but we handle framerate
5915 // adaptation manually by mocking the stats proxy.
5916 video_source_.set_adaptation_enabled(true);
5917
5918 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02005919 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005920 DataRate::BitsPerSec(kTargetBitrateBps),
5921 DataRate::BitsPerSec(kTargetBitrateBps),
5922 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005923 video_stream_encoder_->SetSource(&video_source_,
5924 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005925 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005926
5927 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5928 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
5929
5930 // Pass enough frames with the full update to trigger animation detection.
5931 for (int i = 0; i < kNumFrames; ++i) {
5932 int64_t timestamp_ms =
5933 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5934 frame.set_ntp_time_ms(timestamp_ms);
5935 frame.set_timestamp_us(timestamp_ms * 1000);
5936 video_source_.IncomingCapturedFrame(frame);
5937 WaitForEncodedFrame(timestamp_ms);
5938 }
5939
5940 // Resolution should be limited.
5941 rtc::VideoSinkWants expected;
5942 expected.max_framerate_fps = kFramerateFps;
5943 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005944 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005945
5946 // Pass one frame with no known update.
5947 // Resolution cap should be removed immediately.
5948 int64_t timestamp_ms = fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5949 frame.set_ntp_time_ms(timestamp_ms);
5950 frame.set_timestamp_us(timestamp_ms * 1000);
5951 frame.clear_update_rect();
5952
5953 video_source_.IncomingCapturedFrame(frame);
5954 WaitForEncodedFrame(timestamp_ms);
5955
5956 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005957 EXPECT_THAT(video_source_.sink_wants(),
5958 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01005959
5960 video_stream_encoder_->Stop();
5961}
5962
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02005963TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
5964 const int kWidth = 720; // 540p adapted down.
5965 const int kHeight = 405;
5966 const int kNumFrames = 3;
5967 // Works on screenshare mode.
5968 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5969 /*num_spatial_layers=*/2, /*screenshare=*/true);
5970
5971 video_source_.set_adaptation_enabled(true);
5972
5973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5974 DataRate::BitsPerSec(kTargetBitrateBps),
5975 DataRate::BitsPerSec(kTargetBitrateBps),
5976 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5977
5978 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
5979
5980 // Pass enough frames with the full update to trigger animation detection.
5981 for (int i = 0; i < kNumFrames; ++i) {
5982 int64_t timestamp_ms =
5983 fake_clock_.TimeNanos() / rtc::kNumNanosecsPerMillisec;
5984 frame.set_ntp_time_ms(timestamp_ms);
5985 frame.set_timestamp_us(timestamp_ms * 1000);
5986 video_source_.IncomingCapturedFrame(frame);
5987 WaitForEncodedFrame(timestamp_ms);
5988 }
5989
5990 video_stream_encoder_->Stop();
5991}
5992
perkj26091b12016-09-01 01:17:40 -07005993} // namespace webrtc