blob: 8220daa465a283680d791da3321066e5c04bd5b7 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100321 const VideoStreamEncoderSettings& settings,
322 VideoStreamEncoder::BitrateAllocationCallbackType
323 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200324 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100325 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 stats_proxy,
327 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200328 std::unique_ptr<OveruseFrameDetector>(
329 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100330 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100331 task_queue_factory,
332 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200333 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200334 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200335 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200336 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200337 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200338 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200339 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200340 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100341 }
perkj803d97f2016-11-01 11:45:46 -0700342
Henrik Boström381d1092020-05-12 18:49:07 +0200343 void SetSourceAndWaitForRestrictionsUpdated(
344 rtc::VideoSourceInterface<VideoFrame>* source,
345 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200346 FakeVideoSourceRestrictionsListener listener;
347 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 SetSource(source, degradation_preference);
349 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200350 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200351 }
352
353 void SetSourceAndWaitForFramerateUpdated(
354 rtc::VideoSourceInterface<VideoFrame>* source,
355 const DegradationPreference& degradation_preference) {
356 overuse_detector_proxy_->framerate_updated_event()->Reset();
357 SetSource(source, degradation_preference);
358 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
359 }
360
361 void OnBitrateUpdatedAndWaitForManagedResources(
362 DataRate target_bitrate,
363 DataRate stable_target_bitrate,
364 DataRate link_allocation,
365 uint8_t fraction_lost,
366 int64_t round_trip_time_ms,
367 double cwnd_reduce_ratio) {
368 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
369 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
370 // Bitrate is updated on the encoder queue.
371 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200372 }
373
kthelgason2fc52542017-03-03 00:24:41 -0800374 // This is used as a synchronisation mechanism, to make sure that the
375 // encoder queue is not blocked before we start sending it frames.
376 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100377 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200378 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800379 ASSERT_TRUE(event.Wait(5000));
380 }
381
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200383 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200385 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200386 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200387 event.Set();
388 });
389 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200391 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200392
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 void TriggerCpuUnderuse() {
394 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200395 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200396 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200397 event.Set();
398 });
399 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200400 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 }
kthelgason876222f2016-11-29 01:44:11 -0800402
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200404 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200406 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 event.Set();
409 });
410 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200411 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200412 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200413 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200415 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200416 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 event.Set();
418 });
419 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 }
422
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200423 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100424 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200425 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
426 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200427 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700428};
429
Noah Richards51db4212019-06-12 06:59:12 -0700430// Simulates simulcast behavior and makes highest stream resolutions divisible
431// by 4.
432class CroppingVideoStreamFactory
433 : public VideoEncoderConfig::VideoStreamFactoryInterface {
434 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200435 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700436
437 private:
438 std::vector<VideoStream> CreateEncoderStreams(
439 int width,
440 int height,
441 const VideoEncoderConfig& encoder_config) override {
442 std::vector<VideoStream> streams = test::CreateVideoStreams(
443 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700444 return streams;
445 }
Noah Richards51db4212019-06-12 06:59:12 -0700446};
447
sprangb1ca0732017-02-01 08:38:12 -0800448class AdaptingFrameForwarder : public test::FrameForwarder {
449 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 explicit AdaptingFrameForwarder(TimeController* time_controller)
451 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
asapersson09f05612017-05-15 23:40:18 -0700464 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200465 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700466 return last_wants_;
467 }
468
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200469 absl::optional<int> last_sent_width() const { return last_width_; }
470 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800471
sprangb1ca0732017-02-01 08:38:12 -0800472 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200473 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
474 time_controller_->AdvanceTime(TimeDelta::Millis(0));
475
sprangb1ca0732017-02-01 08:38:12 -0800476 int cropped_width = 0;
477 int cropped_height = 0;
478 int out_width = 0;
479 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700480 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200481 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
482 << "w=" << video_frame.width()
483 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700484 if (adapter_.AdaptFrameResolution(
485 video_frame.width(), video_frame.height(),
486 video_frame.timestamp_us() * 1000, &cropped_width,
487 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100488 VideoFrame adapted_frame =
489 VideoFrame::Builder()
490 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
491 nullptr, out_width, out_height))
492 .set_timestamp_rtp(99)
493 .set_timestamp_ms(99)
494 .set_rotation(kVideoRotation_0)
495 .build();
sprangc5d62e22017-04-02 23:53:04 -0700496 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100497 if (video_frame.has_update_rect()) {
498 adapted_frame.set_update_rect(
499 video_frame.update_rect().ScaleWithFrame(
500 video_frame.width(), video_frame.height(), 0, 0,
501 video_frame.width(), video_frame.height(), out_width,
502 out_height));
503 }
sprangc5d62e22017-04-02 23:53:04 -0700504 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800505 last_width_.emplace(adapted_frame.width());
506 last_height_.emplace(adapted_frame.height());
507 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200508 last_width_ = absl::nullopt;
509 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700510 }
sprangb1ca0732017-02-01 08:38:12 -0800511 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800513 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800514 last_width_.emplace(video_frame.width());
515 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800516 }
517 }
518
519 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
520 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
Markus Handell16038ab2020-05-28 08:37:30 +0200522 last_wants_ = sink_wants_locked();
Rasmus Brandt287e4642019-11-15 16:56:01 +0100523 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200524 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800525 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200526
527 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800528 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200529 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
530 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200531 absl::optional<int> last_width_;
532 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800533};
sprangc5d62e22017-04-02 23:53:04 -0700534
Niels Möller213618e2018-07-24 09:29:58 +0200535// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700536class MockableSendStatisticsProxy : public SendStatisticsProxy {
537 public:
538 MockableSendStatisticsProxy(Clock* clock,
539 const VideoSendStream::Config& config,
540 VideoEncoderConfig::ContentType content_type)
541 : SendStatisticsProxy(clock, config, content_type) {}
542
543 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200544 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700545 if (mock_stats_)
546 return *mock_stats_;
547 return SendStatisticsProxy::GetStats();
548 }
549
Niels Möller213618e2018-07-24 09:29:58 +0200550 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200551 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200552 if (mock_stats_)
553 return mock_stats_->input_frame_rate;
554 return SendStatisticsProxy::GetInputFrameRate();
555 }
sprangc5d62e22017-04-02 23:53:04 -0700556 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200557 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700558 mock_stats_.emplace(stats);
559 }
560
561 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200562 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700563 mock_stats_.reset();
564 }
565
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200566 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
567 on_frame_dropped_ = std::move(callback);
568 }
569
sprangc5d62e22017-04-02 23:53:04 -0700570 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200571 void OnFrameDropped(DropReason reason) override {
572 SendStatisticsProxy::OnFrameDropped(reason);
573 if (on_frame_dropped_)
574 on_frame_dropped_(reason);
575 }
576
Markus Handella3765182020-07-08 13:13:32 +0200577 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200578 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200579 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700580};
581
philipel9b058032020-02-10 11:30:00 +0100582class MockEncoderSelector
583 : public VideoEncoderFactory::EncoderSelectorInterface {
584 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200585 MOCK_METHOD(void,
586 OnCurrentEncoder,
587 (const SdpVideoFormat& format),
588 (override));
589 MOCK_METHOD(absl::optional<SdpVideoFormat>,
590 OnAvailableBitrate,
591 (const DataRate& rate),
592 (override));
593 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100594};
595
perkj803d97f2016-11-01 11:45:46 -0700596} // namespace
597
mflodmancc3d4422017-08-03 08:27:51 -0700598class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700599 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200600 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700601
mflodmancc3d4422017-08-03 08:27:51 -0700602 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700603 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700604 codec_width_(320),
605 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200606 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200608 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700609 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700611 video_send_config_,
612 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700614
615 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700616 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700617 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200618 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800619 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200620 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200621 video_send_config_.rtp.payload_name = "FAKE";
622 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700623
Per512ecb32016-09-23 15:52:06 +0200624 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200625 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200626 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
627 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
628 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100629 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700630
Niels Möllerf1338562018-04-26 09:51:47 +0200631 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800632 }
633
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100634 void ConfigureEncoder(
635 VideoEncoderConfig video_encoder_config,
636 VideoStreamEncoder::BitrateAllocationCallbackType
637 allocation_callback_type =
638 VideoStreamEncoder::BitrateAllocationCallbackType::
639 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700640 if (video_stream_encoder_)
641 video_stream_encoder_->Stop();
642 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200643 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100644 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700645 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700647 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700648 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
649 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200650 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800652 }
653
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100654 void ResetEncoder(const std::string& payload_name,
655 size_t num_streams,
656 size_t num_temporal_layers,
657 unsigned char num_spatial_layers,
658 bool screenshare,
659 VideoStreamEncoder::BitrateAllocationCallbackType
660 allocation_callback_type =
661 VideoStreamEncoder::BitrateAllocationCallbackType::
662 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200663 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800664
665 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200666 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
667 num_streams, &video_encoder_config);
668 for (auto& layer : video_encoder_config.simulcast_layers) {
669 layer.num_temporal_layers = num_temporal_layers;
670 layer.max_framerate = kDefaultFramerate;
671 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100672 video_encoder_config.max_bitrate_bps =
673 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700674 video_encoder_config.content_type =
675 screenshare ? VideoEncoderConfig::ContentType::kScreen
676 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700677 if (payload_name == "VP9") {
678 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
679 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200680 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700681 video_encoder_config.encoder_specific_settings =
682 new rtc::RefCountedObject<
683 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
684 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100685 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700686 }
687
sprang57c2fff2017-01-16 06:24:02 -0800688 VideoFrame CreateFrame(int64_t ntp_time_ms,
689 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100690 VideoFrame frame =
691 VideoFrame::Builder()
692 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
693 destruction_event, codec_width_, codec_height_))
694 .set_timestamp_rtp(99)
695 .set_timestamp_ms(99)
696 .set_rotation(kVideoRotation_0)
697 .build();
sprang57c2fff2017-01-16 06:24:02 -0800698 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700699 return frame;
700 }
701
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100702 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
703 rtc::Event* destruction_event,
704 int offset_x) const {
705 VideoFrame frame =
706 VideoFrame::Builder()
707 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
708 destruction_event, codec_width_, codec_height_))
709 .set_timestamp_rtp(99)
710 .set_timestamp_ms(99)
711 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100712 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100713 .build();
714 frame.set_ntp_time_ms(ntp_time_ms);
715 return frame;
716 }
717
sprang57c2fff2017-01-16 06:24:02 -0800718 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100719 VideoFrame frame =
720 VideoFrame::Builder()
721 .set_video_frame_buffer(
722 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
723 .set_timestamp_rtp(99)
724 .set_timestamp_ms(99)
725 .set_rotation(kVideoRotation_0)
726 .build();
sprang57c2fff2017-01-16 06:24:02 -0800727 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700728 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700729 return frame;
730 }
731
Evan Shrubsole895556e2020-10-05 09:15:13 +0200732 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
733 VideoFrame frame =
734 VideoFrame::Builder()
735 .set_video_frame_buffer(NV12Buffer::Create(width, height))
736 .set_timestamp_rtp(99)
737 .set_timestamp_ms(99)
738 .set_rotation(kVideoRotation_0)
739 .build();
740 frame.set_ntp_time_ms(ntp_time_ms);
741 frame.set_timestamp_us(ntp_time_ms * 1000);
742 return frame;
743 }
744
Noah Richards51db4212019-06-12 06:59:12 -0700745 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int width,
748 int height) const {
749 VideoFrame frame =
750 VideoFrame::Builder()
751 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
752 destruction_event, width, height))
753 .set_timestamp_rtp(99)
754 .set_timestamp_ms(99)
755 .set_rotation(kVideoRotation_0)
756 .build();
757 frame.set_ntp_time_ms(ntp_time_ms);
758 return frame;
759 }
760
Evan Shrubsole895556e2020-10-05 09:15:13 +0200761 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
762 rtc::Event* destruction_event,
763 int width,
764 int height) const {
765 VideoFrame frame = VideoFrame::Builder()
766 .set_video_frame_buffer(
767 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
768 destruction_event, width, height))
769 .set_timestamp_rtp(99)
770 .set_timestamp_ms(99)
771 .set_rotation(kVideoRotation_0)
772 .build();
773 frame.set_ntp_time_ms(ntp_time_ms);
774 return frame;
775 }
776
Noah Richards51db4212019-06-12 06:59:12 -0700777 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
778 rtc::Event* destruction_event) const {
779 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
780 codec_height_);
781 }
782
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100783 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100785 DataRate::BitsPerSec(kTargetBitrateBps),
786 DataRate::BitsPerSec(kTargetBitrateBps),
787 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100788
789 video_source_.IncomingCapturedFrame(
790 CreateFrame(1, codec_width_, codec_height_));
791 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200792 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100793 }
794
sprang4847ae62017-06-27 07:06:52 -0700795 void WaitForEncodedFrame(int64_t expected_ntp_time) {
796 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200797 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700798 }
799
800 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
801 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200802 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700803 return ok;
804 }
805
806 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
807 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200808 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 }
810
811 void ExpectDroppedFrame() {
812 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 }
815
816 bool WaitForFrame(int64_t timeout_ms) {
817 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700819 return ok;
820 }
821
perkj26091b12016-09-01 01:17:40 -0700822 class TestEncoder : public test::FakeEncoder {
823 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 explicit TestEncoder(TimeController* time_controller)
825 : FakeEncoder(time_controller->GetClock()),
826 time_controller_(time_controller) {
827 RTC_DCHECK(time_controller_);
828 }
perkj26091b12016-09-01 01:17:40 -0700829
asaperssonfab67072017-04-04 05:51:49 -0700830 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200831 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700832 return config_;
833 }
834
835 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200836 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700837 block_next_encode_ = true;
838 }
839
Erik Språngaed30702018-11-05 12:57:17 +0100840 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200841 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200842 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100843 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100844 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100845 info.scaling_settings = VideoEncoder::ScalingSettings(
846 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100847 }
848 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100849 for (int i = 0; i < kMaxSpatialLayers; ++i) {
850 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100851 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100852 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100853 for (int tid = 0; tid < num_layers; ++tid)
854 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100855 }
856 }
Erik Språngaed30702018-11-05 12:57:17 +0100857 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200858
859 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100860 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200861 info.apply_alignment_to_all_simulcast_layers =
862 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200863 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100864 return info;
kthelgason876222f2016-11-29 01:44:11 -0800865 }
866
Erik Språngb7cb7b52019-02-26 15:52:33 +0100867 int32_t RegisterEncodeCompleteCallback(
868 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200869 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 encoded_image_callback_ = callback;
871 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
872 }
873
perkjfa10b552016-10-02 23:45:26 -0700874 void ContinueEncode() { continue_encode_event_.Set(); }
875
876 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
877 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700879 EXPECT_EQ(timestamp_, timestamp);
880 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
881 }
882
kthelgason2fc52542017-03-03 00:24:41 -0800883 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200884 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800885 quality_scaling_ = b;
886 }
kthelgasonad9010c2017-02-14 00:46:51 -0800887
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100888 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200889 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100890 requested_resolution_alignment_ = requested_resolution_alignment;
891 }
892
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200893 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
894 MutexLock lock(&local_mutex_);
895 apply_alignment_to_all_simulcast_layers_ = b;
896 }
897
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100898 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100900 is_hardware_accelerated_ = is_hardware_accelerated;
901 }
902
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100903 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
904 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100906 temporal_layers_supported_[spatial_idx] = supported;
907 }
908
Sergey Silkin6456e352019-07-08 17:56:40 +0200909 void SetResolutionBitrateLimits(
910 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200912 resolution_bitrate_limits_ = thresholds;
913 }
914
sprangfe627f32017-03-29 08:24:59 -0700915 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700917 force_init_encode_failed_ = force_failure;
918 }
919
Niels Möller6bb5ab92019-01-11 11:11:10 +0100920 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100922 rate_factor_ = rate_factor;
923 }
924
Erik Språngd7329ca2019-02-21 21:19:53 +0100925 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100927 return last_framerate_;
928 }
929
Erik Språngd7329ca2019-02-21 21:19:53 +0100930 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200931 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100932 return last_update_rect_;
933 }
934
Niels Möller87e2d782019-03-07 10:18:23 +0100935 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200936 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100937 return last_frame_types_;
938 }
939
940 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100941 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100942 keyframe ? VideoFrameType::kVideoFrameKey
943 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100944 {
Markus Handella3765182020-07-08 13:13:32 +0200945 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100946 last_frame_types_ = frame_type;
947 }
Niels Möllerb859b322019-03-07 12:40:01 +0100948 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 }
950
Erik Språngb7cb7b52019-02-26 15:52:33 +0100951 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200953 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100954 }
955
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200956 void SetEncodedImageData(
957 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200958 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200959 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200960 }
961
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 expect_null_frame_ = true;
965 }
966
Erik Språng5056af02019-09-02 15:53:11 +0200967 absl::optional<VideoEncoder::RateControlParameters>
968 GetAndResetLastRateControlSettings() {
969 auto settings = last_rate_control_settings_;
970 last_rate_control_settings_.reset();
971 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 }
973
Evan Shrubsole895556e2020-10-05 09:15:13 +0200974 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
975 MutexLock lock(&local_mutex_);
976 return last_input_pixel_format_;
977 }
978
Sergey Silkin5ee69672019-07-02 14:18:34 +0200979 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200981 return num_encoder_initializations_;
982 }
983
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200984 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200985 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200986 return num_set_rates_;
987 }
988
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200989 VideoCodec video_codec() const {
990 MutexLock lock(&local_mutex_);
991 return video_codec_;
992 }
993
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200994 void SetPreferredPixelFormats(
995 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
996 pixel_formats) {
997 MutexLock lock(&local_mutex_);
998 preferred_pixel_formats_ = std::move(pixel_formats);
999 }
1000
perkjfa10b552016-10-02 23:45:26 -07001001 private:
perkj26091b12016-09-01 01:17:40 -07001002 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001003 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001004 bool block_encode;
1005 {
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001007 if (expect_null_frame_) {
1008 EXPECT_EQ(input_image.timestamp(), 0u);
1009 EXPECT_EQ(input_image.width(), 1);
1010 last_frame_types_ = *frame_types;
1011 expect_null_frame_ = false;
1012 } else {
1013 EXPECT_GT(input_image.timestamp(), timestamp_);
1014 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1015 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1016 }
perkj26091b12016-09-01 01:17:40 -07001017
1018 timestamp_ = input_image.timestamp();
1019 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001020 last_input_width_ = input_image.width();
1021 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001022 block_encode = block_next_encode_;
1023 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001024 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001025 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001026 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001027 }
Niels Möllerb859b322019-03-07 12:40:01 +01001028 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001029 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001030 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031
perkj26091b12016-09-01 01:17:40 -07001032 return result;
1033 }
1034
Niels Möller08ae7ce2020-09-23 15:58:12 +02001035 CodecSpecificInfo EncodeHook(
1036 EncodedImage& encoded_image,
1037 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001038 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001039 {
1040 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001041 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001042 }
1043 MutexLock lock(&local_mutex_);
1044 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001045 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001046 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001047 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001048 }
1049
sprangfe627f32017-03-29 08:24:59 -07001050 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001051 const Settings& settings) override {
1052 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001053
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001055 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001056
1057 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001058 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001059
Erik Språng82fad3d2018-03-21 09:57:23 +01001060 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001061 // Simulate setting up temporal layers, in order to validate the life
1062 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001063 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001064 frame_buffer_controller_ =
1065 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001066 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001067 if (force_init_encode_failed_) {
1068 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001069 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001071
Erik Språngb7cb7b52019-02-26 15:52:33 +01001072 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001073 return res;
1074 }
1075
Erik Språngb7cb7b52019-02-26 15:52:33 +01001076 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001077 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1079 initialized_ = EncoderState::kUninitialized;
1080 return FakeEncoder::Release();
1081 }
1082
Erik Språng16cb8f52019-04-12 13:59:09 +02001083 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001085 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001086 VideoBitrateAllocation adjusted_rate_allocation;
1087 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1088 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001089 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001090 adjusted_rate_allocation.SetBitrate(
1091 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001092 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001093 rate_factor_));
1094 }
1095 }
1096 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001097 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001098 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001099 RateControlParameters adjusted_paramters = parameters;
1100 adjusted_paramters.bitrate = adjusted_rate_allocation;
1101 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001102 }
1103
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001104 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001105 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001106 enum class EncoderState {
1107 kUninitialized,
1108 kInitializationFailed,
1109 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001110 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1111 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001112 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001113 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1114 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1115 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1116 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1117 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1118 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001119 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1120 false;
Markus Handella3765182020-07-08 13:13:32 +02001121 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001122 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1123 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001124 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001125 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001126 absl::optional<bool>
1127 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001128 local_mutex_);
1129 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1130 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1131 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001132 absl::optional<VideoEncoder::RateControlParameters>
1133 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001134 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1135 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001136 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001138 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1139 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001140 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001141 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001142 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001143 RTC_GUARDED_BY(local_mutex_);
1144 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001145 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001146 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1147 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001148 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1149 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001150 };
1151
mflodmancc3d4422017-08-03 08:27:51 -07001152 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001153 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001154 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1155 : time_controller_(time_controller), test_encoder_(test_encoder) {
1156 RTC_DCHECK(time_controller_);
1157 }
perkj26091b12016-09-01 01:17:40 -07001158
perkj26091b12016-09-01 01:17:40 -07001159 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001160 EXPECT_TRUE(
1161 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1162 }
1163
1164 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1165 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001166 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001167 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001168 return false;
perkj26091b12016-09-01 01:17:40 -07001169 {
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001171 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001172 }
1173 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001174 return true;
perkj26091b12016-09-01 01:17:40 -07001175 }
1176
sprangb1ca0732017-02-01 08:38:12 -08001177 void WaitForEncodedFrame(uint32_t expected_width,
1178 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001179 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001180 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001181 }
1182
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001183 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001184 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001185 uint32_t width = 0;
1186 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001187 {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001189 width = last_width_;
1190 height = last_height_;
1191 }
1192 EXPECT_EQ(expected_height, height);
1193 EXPECT_EQ(expected_width, width);
1194 }
1195
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001196 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1197 VideoRotation rotation;
1198 {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001200 rotation = last_rotation_;
1201 }
1202 EXPECT_EQ(expected_rotation, rotation);
1203 }
1204
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001205 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001206
sprangc5d62e22017-04-02 23:53:04 -07001207 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001208 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1209 bool ret = encoded_frame_event_.Wait(timeout_ms);
1210 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1211 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001212 }
1213
perkj26091b12016-09-01 01:17:40 -07001214 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001215 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001216 expect_frames_ = false;
1217 }
1218
asaperssonfab67072017-04-04 05:51:49 -07001219 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001220 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001221 return number_of_reconfigurations_;
1222 }
1223
asaperssonfab67072017-04-04 05:51:49 -07001224 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001225 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001226 return min_transmit_bitrate_bps_;
1227 }
1228
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001230 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001231 num_expected_layers_ = num_layers;
1232 }
1233
Erik Språngb7cb7b52019-02-26 15:52:33 +01001234 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001235 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001236 return last_capture_time_ms_;
1237 }
1238
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001239 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001240 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001241 return std::move(last_encoded_image_data_);
1242 }
1243
Per Kjellanderdcef6412020-10-07 15:09:05 +02001244 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1245 MutexLock lock(&mutex_);
1246 return last_bitrate_allocation_;
1247 }
1248
1249 int number_of_bitrate_allocations() const {
1250 MutexLock lock(&mutex_);
1251 return number_of_bitrate_allocations_;
1252 }
1253
Per Kjellandera9434842020-10-15 17:53:22 +02001254 VideoLayersAllocation GetLastVideoLayersAllocation() {
1255 MutexLock lock(&mutex_);
1256 return last_layers_allocation_;
1257 }
1258
1259 int number_of_layers_allocations() const {
1260 MutexLock lock(&mutex_);
1261 return number_of_layers_allocations_;
1262 }
1263
perkj26091b12016-09-01 01:17:40 -07001264 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001265 Result OnEncodedImage(
1266 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001267 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001269 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001270 last_encoded_image_data_ = std::vector<uint8_t>(
1271 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001272 uint32_t timestamp = encoded_image.Timestamp();
1273 if (last_timestamp_ != timestamp) {
1274 num_received_layers_ = 1;
1275 } else {
1276 ++num_received_layers_;
1277 }
1278 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001279 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001280 last_width_ = encoded_image._encodedWidth;
1281 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001282 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001283 if (num_received_layers_ == num_expected_layers_) {
1284 encoded_frame_event_.Set();
1285 }
sprangb1ca0732017-02-01 08:38:12 -08001286 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001287 }
1288
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001289 void OnEncoderConfigurationChanged(
1290 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001291 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001292 VideoEncoderConfig::ContentType content_type,
1293 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001294 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001295 ++number_of_reconfigurations_;
1296 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1297 }
1298
Per Kjellanderdcef6412020-10-07 15:09:05 +02001299 void OnBitrateAllocationUpdated(
1300 const VideoBitrateAllocation& allocation) override {
1301 MutexLock lock(&mutex_);
1302 ++number_of_bitrate_allocations_;
1303 last_bitrate_allocation_ = allocation;
1304 }
1305
Per Kjellandera9434842020-10-15 17:53:22 +02001306 void OnVideoLayersAllocationUpdated(
1307 VideoLayersAllocation allocation) override {
1308 MutexLock lock(&mutex_);
1309 ++number_of_layers_allocations_;
1310 last_layers_allocation_ = allocation;
1311 rtc::StringBuilder log;
1312 for (const auto& layer : allocation.active_spatial_layers) {
1313 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1314 << "[";
1315 for (const auto target_bitrate :
1316 layer.target_bitrate_per_temporal_layer) {
1317 log << target_bitrate.kbps() << ",";
1318 }
1319 log << "]";
1320 }
1321 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1322 }
1323
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001324 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001325 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001326 TestEncoder* test_encoder_;
1327 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001328 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001329 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001330 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001331 uint32_t last_height_ = 0;
1332 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001333 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001334 size_t num_expected_layers_ = 1;
1335 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001336 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001337 int number_of_reconfigurations_ = 0;
1338 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001339 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1340 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001341 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1342 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001343 };
1344
Sergey Silkin5ee69672019-07-02 14:18:34 +02001345 class VideoBitrateAllocatorProxyFactory
1346 : public VideoBitrateAllocatorFactory {
1347 public:
1348 VideoBitrateAllocatorProxyFactory()
1349 : bitrate_allocator_factory_(
1350 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1351
1352 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1353 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001354 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001355 codec_config_ = codec;
1356 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1357 }
1358
1359 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001360 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001361 return codec_config_;
1362 }
1363
1364 private:
1365 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1366
Markus Handella3765182020-07-08 13:13:32 +02001367 mutable Mutex mutex_;
1368 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001369 };
1370
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001371 Clock* clock() { return time_controller_.GetClock(); }
1372 void AdvanceTime(TimeDelta duration) {
1373 time_controller_.AdvanceTime(duration);
1374 }
1375
1376 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1377
1378 protected:
1379 virtual TaskQueueFactory* GetTaskQueueFactory() {
1380 return time_controller_.GetTaskQueueFactory();
1381 }
1382
1383 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001384 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001385 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001386 int codec_width_;
1387 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001388 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001389 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001390 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001391 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001392 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001393 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001394 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001395 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001396};
1397
mflodmancc3d4422017-08-03 08:27:51 -07001398TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001400 DataRate::BitsPerSec(kTargetBitrateBps),
1401 DataRate::BitsPerSec(kTargetBitrateBps),
1402 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001403 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001404 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001405 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001406 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001407 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001408}
1409
mflodmancc3d4422017-08-03 08:27:51 -07001410TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001411 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001412 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001413 // The encoder will cache up to one frame for a short duration. Adding two
1414 // frames means that the first frame will be dropped and the second frame will
1415 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001416 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001417 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001418 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001419 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001420
Henrik Boström381d1092020-05-12 18:49:07 +02001421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001422 DataRate::BitsPerSec(kTargetBitrateBps),
1423 DataRate::BitsPerSec(kTargetBitrateBps),
1424 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001425
Sebastian Janssona3177052018-04-10 13:05:49 +02001426 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001427 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001428 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1429
1430 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001431 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001432}
1433
mflodmancc3d4422017-08-03 08:27:51 -07001434TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001436 DataRate::BitsPerSec(kTargetBitrateBps),
1437 DataRate::BitsPerSec(kTargetBitrateBps),
1438 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001439 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001440 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001441
Henrik Boström381d1092020-05-12 18:49:07 +02001442 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1443 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1444 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001445 // The encoder will cache up to one frame for a short duration. Adding two
1446 // frames means that the first frame will be dropped and the second frame will
1447 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001448 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001449 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001450
Henrik Boström381d1092020-05-12 18:49:07 +02001451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001452 DataRate::BitsPerSec(kTargetBitrateBps),
1453 DataRate::BitsPerSec(kTargetBitrateBps),
1454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001455 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1457 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001458 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001459}
1460
mflodmancc3d4422017-08-03 08:27:51 -07001461TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001466 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001467 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001468
1469 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001470 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001471
perkja49cbd32016-09-16 07:53:41 -07001472 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001473 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001474 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001475}
1476
mflodmancc3d4422017-08-03 08:27:51 -07001477TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps),
1481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001482
perkja49cbd32016-09-16 07:53:41 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001484 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001485
mflodmancc3d4422017-08-03 08:27:51 -07001486 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001487 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001488 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001489 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1490 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001491}
1492
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001493class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1494 public:
1495 VideoStreamEncoderBlockedTest() {}
1496
1497 TaskQueueFactory* GetTaskQueueFactory() override {
1498 return task_queue_factory_.get();
1499 }
1500
1501 private:
1502 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1503 CreateDefaultTaskQueueFactory();
1504};
1505
1506TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps),
1510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001511
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001512 int dropped_count = 0;
1513 stats_proxy_->SetDroppedFrameCallback(
1514 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1515 ++dropped_count;
1516 });
1517
perkj26091b12016-09-01 01:17:40 -07001518 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001519 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001520 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001521 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1522 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001523 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1524 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001525 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001526 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001527
mflodmancc3d4422017-08-03 08:27:51 -07001528 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001529
1530 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001531}
1532
Noah Richards51db4212019-06-12 06:59:12 -07001533TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001538
1539 rtc::Event frame_destroyed_event;
1540 video_source_.IncomingCapturedFrame(
1541 CreateFakeNativeFrame(1, &frame_destroyed_event));
1542 ExpectDroppedFrame();
1543 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1544 video_stream_encoder_->Stop();
1545}
1546
1547TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1548 // Use the cropping factory.
1549 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001550 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001551 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1552 kMaxPayloadLength);
1553 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1554
1555 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001557 DataRate::BitsPerSec(kTargetBitrateBps),
1558 DataRate::BitsPerSec(kTargetBitrateBps),
1559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001560 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1561 WaitForEncodedFrame(1);
1562 // The encoder will have been configured once.
1563 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1564 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1565 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1566
1567 // Now send in a fake frame that needs to be cropped as the width/height
1568 // aren't divisible by 4 (see CreateEncoderStreams above).
1569 rtc::Event frame_destroyed_event;
1570 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1571 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1572 ExpectDroppedFrame();
1573 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1574 video_stream_encoder_->Stop();
1575}
1576
Evan Shrubsole895556e2020-10-05 09:15:13 +02001577TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1579 DataRate::BitsPerSec(kTargetBitrateBps),
1580 DataRate::BitsPerSec(kTargetBitrateBps),
1581 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1582
1583 video_source_.IncomingCapturedFrame(
1584 CreateNV12Frame(1, codec_width_, codec_height_));
1585 WaitForEncodedFrame(1);
1586 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1587 fake_encoder_.GetLastInputPixelFormat());
1588 video_stream_encoder_->Stop();
1589}
1590
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001591TEST_F(VideoStreamEncoderTest,
1592 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1594 DataRate::BitsPerSec(kTargetBitrateBps),
1595 DataRate::BitsPerSec(kTargetBitrateBps),
1596 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1597
1598 fake_encoder_.SetPreferredPixelFormats({});
1599
1600 rtc::Event frame_destroyed_event;
1601 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1602 1, &frame_destroyed_event, codec_width_, codec_height_));
1603 WaitForEncodedFrame(1);
1604 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1605 fake_encoder_.GetLastInputPixelFormat());
1606 video_stream_encoder_->Stop();
1607}
1608
1609TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1611 DataRate::BitsPerSec(kTargetBitrateBps),
1612 DataRate::BitsPerSec(kTargetBitrateBps),
1613 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1614
1615 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1616
1617 rtc::Event frame_destroyed_event;
1618 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1619 1, &frame_destroyed_event, codec_width_, codec_height_));
1620 WaitForEncodedFrame(1);
1621 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1622 fake_encoder_.GetLastInputPixelFormat());
1623 video_stream_encoder_->Stop();
1624}
1625
1626TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1628 DataRate::BitsPerSec(kTargetBitrateBps),
1629 DataRate::BitsPerSec(kTargetBitrateBps),
1630 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1631
1632 // Fake NV12 native frame does not allow mapping to I444.
1633 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1634
1635 rtc::Event frame_destroyed_event;
1636 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1637 1, &frame_destroyed_event, codec_width_, codec_height_));
1638 WaitForEncodedFrame(1);
1639 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1640 fake_encoder_.GetLastInputPixelFormat());
1641 video_stream_encoder_->Stop();
1642}
1643
Evan Shrubsole895556e2020-10-05 09:15:13 +02001644TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1646 DataRate::BitsPerSec(kTargetBitrateBps),
1647 DataRate::BitsPerSec(kTargetBitrateBps),
1648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1649
1650 rtc::Event frame_destroyed_event;
1651 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1652 1, &frame_destroyed_event, codec_width_, codec_height_));
1653 WaitForEncodedFrame(1);
1654 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1655 fake_encoder_.GetLastInputPixelFormat());
1656 video_stream_encoder_->Stop();
1657}
1658
Ying Wang9b881ab2020-02-07 14:29:32 +01001659TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps),
1663 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001664 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1665 WaitForEncodedFrame(1);
1666
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001668 DataRate::BitsPerSec(kTargetBitrateBps),
1669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001671 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1672 // frames. Adding two frames means that the first frame will be dropped and
1673 // the second frame will be sent to the encoder.
1674 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1675 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1676 WaitForEncodedFrame(3);
1677 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1678 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1679 WaitForEncodedFrame(5);
1680 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1681 video_stream_encoder_->Stop();
1682}
1683
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest,
1685 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps),
1689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001690 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001691
1692 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001693 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001694 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001695 // The encoder will have been configured once when the first frame is
1696 // received.
1697 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001698
1699 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001700 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001701 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001702 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001703 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001704
1705 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001706 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001707 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001708 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001709 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001710
mflodmancc3d4422017-08-03 08:27:51 -07001711 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001712}
1713
mflodmancc3d4422017-08-03 08:27:51 -07001714TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001716 DataRate::BitsPerSec(kTargetBitrateBps),
1717 DataRate::BitsPerSec(kTargetBitrateBps),
1718 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001719
1720 // Capture a frame and wait for it to synchronize with the encoder thread.
1721 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001722 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001723 // The encoder will have been configured once.
1724 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001725 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1726 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1727
1728 codec_width_ *= 2;
1729 codec_height_ *= 2;
1730 // Capture a frame with a higher resolution and wait for it to synchronize
1731 // with the encoder thread.
1732 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001733 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001734 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1735 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001736 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001737
mflodmancc3d4422017-08-03 08:27:51 -07001738 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001739}
1740
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001741TEST_F(VideoStreamEncoderTest,
1742 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001744 DataRate::BitsPerSec(kTargetBitrateBps),
1745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001747
1748 // Capture a frame and wait for it to synchronize with the encoder thread.
1749 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1750 WaitForEncodedFrame(1);
1751
1752 VideoEncoderConfig video_encoder_config;
1753 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1754 // Changing the max payload data length recreates encoder.
1755 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1756 kMaxPayloadLength / 2);
1757
1758 // Capture a frame and wait for it to synchronize with the encoder thread.
1759 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1760 WaitForEncodedFrame(2);
1761 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1762
1763 video_stream_encoder_->Stop();
1764}
1765
Sergey Silkin5ee69672019-07-02 14:18:34 +02001766TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001768 DataRate::BitsPerSec(kTargetBitrateBps),
1769 DataRate::BitsPerSec(kTargetBitrateBps),
1770 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001771
1772 VideoEncoderConfig video_encoder_config;
1773 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1774 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1775 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1776 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1777 kMaxPayloadLength);
1778
1779 // Capture a frame and wait for it to synchronize with the encoder thread.
1780 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1781 WaitForEncodedFrame(1);
1782 // The encoder will have been configured once when the first frame is
1783 // received.
1784 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1785 EXPECT_EQ(kTargetBitrateBps,
1786 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1787 EXPECT_EQ(kStartBitrateBps,
1788 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1789
Sergey Silkin6456e352019-07-08 17:56:40 +02001790 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1791 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001792 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1793 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1794 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1795 kMaxPayloadLength);
1796
1797 // Capture a frame and wait for it to synchronize with the encoder thread.
1798 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1799 WaitForEncodedFrame(2);
1800 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1801 // Bitrate limits have changed - rate allocator should be reconfigured,
1802 // encoder should not be reconfigured.
1803 EXPECT_EQ(kTargetBitrateBps * 2,
1804 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1805 EXPECT_EQ(kStartBitrateBps * 2,
1806 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1807 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1808
1809 video_stream_encoder_->Stop();
1810}
1811
Sergey Silkin6456e352019-07-08 17:56:40 +02001812TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001813 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001814 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001815 DataRate::BitsPerSec(kTargetBitrateBps),
1816 DataRate::BitsPerSec(kTargetBitrateBps),
1817 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001818
Sergey Silkincd02eba2020-01-20 14:48:40 +01001819 const uint32_t kMinEncBitrateKbps = 100;
1820 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001821 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001822 /*frame_size_pixels=*/codec_width_ * codec_height_,
1823 /*min_start_bitrate_bps=*/0,
1824 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1825 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001826 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1827
Sergey Silkincd02eba2020-01-20 14:48:40 +01001828 VideoEncoderConfig video_encoder_config;
1829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1830 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1831 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1832 (kMinEncBitrateKbps + 1) * 1000;
1833 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1834 kMaxPayloadLength);
1835
1836 // When both encoder and app provide bitrate limits, the intersection of
1837 // provided sets should be used.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 EXPECT_EQ(kMaxEncBitrateKbps,
1841 bitrate_allocator_factory_.codec_config().maxBitrate);
1842 EXPECT_EQ(kMinEncBitrateKbps + 1,
1843 bitrate_allocator_factory_.codec_config().minBitrate);
1844
1845 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1846 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1847 (kMinEncBitrateKbps - 1) * 1000;
1848 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1849 kMaxPayloadLength);
1850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001851 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001852 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001853 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001854 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001855 bitrate_allocator_factory_.codec_config().minBitrate);
1856
Sergey Silkincd02eba2020-01-20 14:48:40 +01001857 video_stream_encoder_->Stop();
1858}
1859
1860TEST_F(VideoStreamEncoderTest,
1861 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001863 DataRate::BitsPerSec(kTargetBitrateBps),
1864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001866
1867 const uint32_t kMinAppBitrateKbps = 100;
1868 const uint32_t kMaxAppBitrateKbps = 200;
1869 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1870 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1871 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1872 /*frame_size_pixels=*/codec_width_ * codec_height_,
1873 /*min_start_bitrate_bps=*/0,
1874 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1875 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1876 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1877
1878 VideoEncoderConfig video_encoder_config;
1879 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1880 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1881 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1882 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1884 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001885
Sergey Silkincd02eba2020-01-20 14:48:40 +01001886 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1887 WaitForEncodedFrame(1);
1888 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001889 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001890 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001891 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001892
1893 video_stream_encoder_->Stop();
1894}
1895
1896TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001897 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001899 DataRate::BitsPerSec(kTargetBitrateBps),
1900 DataRate::BitsPerSec(kTargetBitrateBps),
1901 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001902
1903 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001904 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001905 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001906 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001907 fake_encoder_.SetResolutionBitrateLimits(
1908 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1909
1910 VideoEncoderConfig video_encoder_config;
1911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1912 video_encoder_config.max_bitrate_bps = 0;
1913 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1914 kMaxPayloadLength);
1915
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001916 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1918 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1920 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001921 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1922 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1923
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001924 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001925 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1926 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001927 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1928 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001929 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1930 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1931
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001933 // encoder for 360p should be used.
1934 video_source_.IncomingCapturedFrame(
1935 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1936 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001937 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1938 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001939 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1940 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1941
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001943 // ignored.
1944 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1945 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001946 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001948 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1949 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001950 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1951 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001952 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1953 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1954
1955 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1956 // for 270p should be used.
1957 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1958 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001959 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1960 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001961 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1962 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1963
1964 video_stream_encoder_->Stop();
1965}
1966
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001967TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001969 DataRate::BitsPerSec(kTargetBitrateBps),
1970 DataRate::BitsPerSec(kTargetBitrateBps),
1971 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972
1973 VideoEncoderConfig video_encoder_config;
1974 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1975 video_encoder_config.max_bitrate_bps = 0;
1976 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1977 kMaxPayloadLength);
1978
1979 // Encode 720p frame to get the default encoder target bitrate.
1980 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1981 WaitForEncodedFrame(1);
1982 const uint32_t kDefaultTargetBitrateFor720pKbps =
1983 bitrate_allocator_factory_.codec_config()
1984 .simulcastStream[0]
1985 .targetBitrate;
1986
1987 // Set the max recommended encoder bitrate to something lower than the default
1988 // target bitrate.
1989 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1990 1280 * 720, 10 * 1000, 10 * 1000,
1991 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1992 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1993
1994 // Change resolution to trigger encoder reinitialization.
1995 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1996 WaitForEncodedFrame(2);
1997 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1998 WaitForEncodedFrame(3);
1999
2000 // Ensure the target bitrate is capped by the max bitrate.
2001 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2002 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2003 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2004 .simulcastStream[0]
2005 .targetBitrate *
2006 1000,
2007 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2008
2009 video_stream_encoder_->Stop();
2010}
2011
Åsa Perssona7e34d32021-01-20 15:36:13 +01002012TEST_F(VideoStreamEncoderTest,
2013 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2014 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2015 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2016 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2017 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2018 fake_encoder_.SetResolutionBitrateLimits(
2019 {kEncoderLimits270p, kEncoderLimits360p});
2020
2021 // Two streams, highest stream active.
2022 VideoEncoderConfig config;
2023 const int kNumStreams = 2;
2024 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2025 config.max_bitrate_bps = 0;
2026 config.simulcast_layers[0].active = false;
2027 config.simulcast_layers[1].active = true;
2028 config.video_stream_factory =
2029 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2030 "VP8", /*max qp*/ 56, /*screencast*/ false,
2031 /*screenshare enabled*/ false);
2032 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2033
2034 // The encoder bitrate limits for 270p should be used.
2035 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2036 EXPECT_FALSE(WaitForFrame(1000));
2037 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2038 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2039 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2040 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2041 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2042
2043 // The encoder bitrate limits for 360p should be used.
2044 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2045 EXPECT_FALSE(WaitForFrame(1000));
2046 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2047 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2048 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2049 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2050
2051 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2052 video_source_.IncomingCapturedFrame(
2053 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2054 EXPECT_FALSE(WaitForFrame(1000));
2055 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2056 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2057 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2058 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2059
2060 // Resolution higher than 360p. Encoder limits should be ignored.
2061 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2062 EXPECT_FALSE(WaitForFrame(1000));
2063 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2064 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2065 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2066 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2067 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2068 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2069 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2070 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2071
2072 // Resolution lower than 270p. The encoder limits for 270p should be used.
2073 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2074 EXPECT_FALSE(WaitForFrame(1000));
2075 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2076 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2077 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2078 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2079
2080 video_stream_encoder_->Stop();
2081}
2082
2083TEST_F(VideoStreamEncoderTest,
2084 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2085 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2086 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2087 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2088 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2089 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2090 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2091 fake_encoder_.SetResolutionBitrateLimits(
2092 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2093
2094 // Three streams, middle stream active.
2095 VideoEncoderConfig config;
2096 const int kNumStreams = 3;
2097 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2098 config.simulcast_layers[0].active = false;
2099 config.simulcast_layers[1].active = true;
2100 config.simulcast_layers[2].active = false;
2101 config.video_stream_factory =
2102 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2103 "VP8", /*max qp*/ 56, /*screencast*/ false,
2104 /*screenshare enabled*/ false);
2105 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2106
2107 // The encoder bitrate limits for 360p should be used.
2108 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2109 EXPECT_FALSE(WaitForFrame(1000));
2110 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2111 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2112 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2114 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2115
2116 // The encoder bitrate limits for 270p should be used.
2117 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2118 EXPECT_FALSE(WaitForFrame(1000));
2119 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2120 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2121 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2122 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2123
2124 video_stream_encoder_->Stop();
2125}
2126
2127TEST_F(VideoStreamEncoderTest,
2128 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2129 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2130 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2131 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2132 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2133 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2134 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2135 fake_encoder_.SetResolutionBitrateLimits(
2136 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2137
2138 // Three streams, lowest stream active.
2139 VideoEncoderConfig config;
2140 const int kNumStreams = 3;
2141 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2142 config.simulcast_layers[0].active = true;
2143 config.simulcast_layers[1].active = false;
2144 config.simulcast_layers[2].active = false;
2145 config.video_stream_factory =
2146 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2147 "VP8", /*max qp*/ 56, /*screencast*/ false,
2148 /*screenshare enabled*/ false);
2149 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2150
2151 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2152 // on lowest stream, limits for 270p should not be used
2153 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2154 EXPECT_FALSE(WaitForFrame(1000));
2155 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2156 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2157 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2158 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2159 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2160
2161 video_stream_encoder_->Stop();
2162}
2163
2164TEST_F(VideoStreamEncoderTest,
2165 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2166 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2167 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2168 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2169 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2170 fake_encoder_.SetResolutionBitrateLimits(
2171 {kEncoderLimits270p, kEncoderLimits360p});
2172 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2173
2174 // Two streams, highest stream active.
2175 VideoEncoderConfig config;
2176 const int kNumStreams = 2;
2177 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2178 config.simulcast_layers[0].active = false;
2179 config.simulcast_layers[1].active = true;
2180 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2181 config.video_stream_factory =
2182 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2183 "VP8", /*max qp*/ 56, /*screencast*/ false,
2184 /*screenshare enabled*/ false);
2185 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2186
2187 // The encoder bitrate limits for 270p should be used.
2188 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2191 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2192 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2193 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2194 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2195
2196 // The max configured bitrate is less than the encoder limit for 360p.
2197 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2198 EXPECT_FALSE(WaitForFrame(1000));
2199 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2200 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2201 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2202 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2203
2204 video_stream_encoder_->Stop();
2205}
2206
mflodmancc3d4422017-08-03 08:27:51 -07002207TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002208 EXPECT_TRUE(video_source_.has_sinks());
2209 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002210 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002211 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002212 EXPECT_FALSE(video_source_.has_sinks());
2213 EXPECT_TRUE(new_video_source.has_sinks());
2214
mflodmancc3d4422017-08-03 08:27:51 -07002215 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002216}
2217
mflodmancc3d4422017-08-03 08:27:51 -07002218TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002219 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002220 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002221 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002222 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002223}
2224
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002225class ResolutionAlignmentTest
2226 : public VideoStreamEncoderTest,
2227 public ::testing::WithParamInterface<
2228 ::testing::tuple<int, std::vector<double>>> {
2229 public:
2230 ResolutionAlignmentTest()
2231 : requested_alignment_(::testing::get<0>(GetParam())),
2232 scale_factors_(::testing::get<1>(GetParam())) {}
2233
2234 protected:
2235 const int requested_alignment_;
2236 const std::vector<double> scale_factors_;
2237};
2238
2239INSTANTIATE_TEST_SUITE_P(
2240 AlignmentAndScaleFactors,
2241 ResolutionAlignmentTest,
2242 ::testing::Combine(
2243 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2244 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2245 std::vector<double>{-1.0, -1.0},
2246 std::vector<double>{-1.0, -1.0, -1.0},
2247 std::vector<double>{4.0, 2.0, 1.0},
2248 std::vector<double>{9999.0, -1.0, 1.0},
2249 std::vector<double>{3.99, 2.01, 1.0},
2250 std::vector<double>{4.9, 1.7, 1.25},
2251 std::vector<double>{10.0, 4.0, 3.0},
2252 std::vector<double>{1.75, 3.5},
2253 std::vector<double>{1.5, 2.5},
2254 std::vector<double>{1.3, 1.0})));
2255
2256TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2257 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002258 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002259 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2260 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2261
2262 // Fill config with the scaling factor by which to reduce encoding size.
2263 const int num_streams = scale_factors_.size();
2264 VideoEncoderConfig config;
2265 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2266 for (int i = 0; i < num_streams; ++i) {
2267 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2268 }
2269 config.video_stream_factory =
2270 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2271 "VP8", /*max qp*/ 56, /*screencast*/ false,
2272 /*screenshare enabled*/ false);
2273 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2274
Henrik Boström381d1092020-05-12 18:49:07 +02002275 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002276 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2277 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2278 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2279 // Wait for all layers before triggering event.
2280 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002281
2282 // On the 1st frame, we should have initialized the encoder and
2283 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002284 int64_t timestamp_ms = kFrameIntervalMs;
2285 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2286 WaitForEncodedFrame(timestamp_ms);
2287 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002288
2289 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2290 // (It's up the to the encoder to potentially drop the previous frame,
2291 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002292 timestamp_ms += kFrameIntervalMs;
2293 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2294 WaitForEncodedFrame(timestamp_ms);
2295 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2296
2297 VideoCodec codec = fake_encoder_.video_codec();
2298 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2299 // Frame size should be a multiple of the requested alignment.
2300 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2301 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2302 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2303 // Aspect ratio should match.
2304 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2305 codec.height * codec.simulcastStream[i].width);
2306 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002307
2308 video_stream_encoder_->Stop();
2309}
2310
Jonathan Yubc771b72017-12-08 17:04:29 -08002311TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2312 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002313 const int kWidth = 1280;
2314 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002315
2316 // We rely on the automatic resolution adaptation, but we handle framerate
2317 // adaptation manually by mocking the stats proxy.
2318 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002319
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002320 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002322 DataRate::BitsPerSec(kTargetBitrateBps),
2323 DataRate::BitsPerSec(kTargetBitrateBps),
2324 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002325 video_stream_encoder_->SetSource(&video_source_,
2326 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002327 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002328 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002329 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002330 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2331
Jonathan Yubc771b72017-12-08 17:04:29 -08002332 // Adapt down as far as possible.
2333 rtc::VideoSinkWants last_wants;
2334 int64_t t = 1;
2335 int loop_count = 0;
2336 do {
2337 ++loop_count;
2338 last_wants = video_source_.sink_wants();
2339
2340 // Simulate the framerate we've been asked to adapt to.
2341 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2342 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2343 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2344 mock_stats.input_frame_rate = fps;
2345 stats_proxy_->SetMockStats(mock_stats);
2346
2347 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2348 sink_.WaitForEncodedFrame(t);
2349 t += frame_interval_ms;
2350
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002352 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002353 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002354 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2355 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002356 } while (video_source_.sink_wants().max_pixel_count <
2357 last_wants.max_pixel_count ||
2358 video_source_.sink_wants().max_framerate_fps <
2359 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002360
Jonathan Yubc771b72017-12-08 17:04:29 -08002361 // Verify that we've adapted all the way down.
2362 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002363 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002364 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2365 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002366 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002367 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2368 *video_source_.last_sent_height());
2369 EXPECT_EQ(kMinBalancedFramerateFps,
2370 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002371
Jonathan Yubc771b72017-12-08 17:04:29 -08002372 // Adapt back up the same number of times we adapted down.
2373 for (int i = 0; i < loop_count - 1; ++i) {
2374 last_wants = video_source_.sink_wants();
2375
2376 // Simulate the framerate we've been asked to adapt to.
2377 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2378 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2379 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2380 mock_stats.input_frame_rate = fps;
2381 stats_proxy_->SetMockStats(mock_stats);
2382
2383 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2384 sink_.WaitForEncodedFrame(t);
2385 t += frame_interval_ms;
2386
Henrik Boström91aa7322020-04-28 12:24:33 +02002387 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002388 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002389 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002390 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2391 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002392 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2393 last_wants.max_pixel_count ||
2394 video_source_.sink_wants().max_framerate_fps >
2395 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002396 }
2397
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002398 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002399 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002400 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002401 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2402 EXPECT_EQ((loop_count - 1) * 2,
2403 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002406}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002407
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002408TEST_F(VideoStreamEncoderTest,
2409 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2410 video_stream_encoder_->OnBitrateUpdated(
2411 DataRate::BitsPerSec(kTargetBitrateBps),
2412 DataRate::BitsPerSec(kTargetBitrateBps),
2413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002414 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002415
2416 const int kFrameWidth = 1280;
2417 const int kFrameHeight = 720;
2418
2419 int64_t ntp_time = kFrameIntervalMs;
2420
2421 // Force an input frame rate to be available, or the adaptation call won't
2422 // know what framerate to adapt form.
2423 const int kInputFps = 30;
2424 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2425 stats.input_frame_rate = kInputFps;
2426 stats_proxy_->SetMockStats(stats);
2427
2428 video_source_.set_adaptation_enabled(true);
2429 video_stream_encoder_->SetSource(
2430 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002431 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002432 video_source_.IncomingCapturedFrame(
2433 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2434 sink_.WaitForEncodedFrame(ntp_time);
2435 ntp_time += kFrameIntervalMs;
2436
2437 // Trigger CPU overuse.
2438 video_stream_encoder_->TriggerCpuOveruse();
2439 video_source_.IncomingCapturedFrame(
2440 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2441 sink_.WaitForEncodedFrame(ntp_time);
2442 ntp_time += kFrameIntervalMs;
2443
2444 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2445 EXPECT_EQ(std::numeric_limits<int>::max(),
2446 video_source_.sink_wants().max_pixel_count);
2447 // Some framerate constraint should be set.
2448 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2449 EXPECT_LT(restricted_fps, kInputFps);
2450 video_source_.IncomingCapturedFrame(
2451 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2452 sink_.WaitForEncodedFrame(ntp_time);
2453 ntp_time += 100;
2454
Henrik Boström2671dac2020-05-19 16:29:09 +02002455 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002456 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2457 // Give the encoder queue time to process the change in degradation preference
2458 // by waiting for an encoded frame.
2459 video_source_.IncomingCapturedFrame(
2460 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2461 sink_.WaitForEncodedFrame(ntp_time);
2462 ntp_time += kFrameIntervalMs;
2463
2464 video_stream_encoder_->TriggerQualityLow();
2465 video_source_.IncomingCapturedFrame(
2466 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2467 sink_.WaitForEncodedFrame(ntp_time);
2468 ntp_time += kFrameIntervalMs;
2469
2470 // Some resolution constraint should be set.
2471 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2472 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2473 kFrameWidth * kFrameHeight);
2474 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2475
2476 int pixel_count = video_source_.sink_wants().max_pixel_count;
2477 // Triggering a CPU underuse should not change the sink wants since it has
2478 // not been overused for resolution since we changed degradation preference.
2479 video_stream_encoder_->TriggerCpuUnderuse();
2480 video_source_.IncomingCapturedFrame(
2481 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2482 sink_.WaitForEncodedFrame(ntp_time);
2483 ntp_time += kFrameIntervalMs;
2484 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2485 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2486
Evan Shrubsole64469032020-06-11 10:45:29 +02002487 // Change the degradation preference back. CPU underuse should not adapt since
2488 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002489 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002490 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2491 video_source_.IncomingCapturedFrame(
2492 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2493 sink_.WaitForEncodedFrame(ntp_time);
2494 ntp_time += 100;
2495 // Resolution adaptations is gone after changing degradation preference.
2496 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2497 EXPECT_EQ(std::numeric_limits<int>::max(),
2498 video_source_.sink_wants().max_pixel_count);
2499 // The fps adaptation from above is now back.
2500 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2501
2502 // Trigger CPU underuse.
2503 video_stream_encoder_->TriggerCpuUnderuse();
2504 video_source_.IncomingCapturedFrame(
2505 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2506 sink_.WaitForEncodedFrame(ntp_time);
2507 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002508 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2509
2510 // Trigger QP underuse, fps should return to normal.
2511 video_stream_encoder_->TriggerQualityHigh();
2512 video_source_.IncomingCapturedFrame(
2513 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2514 sink_.WaitForEncodedFrame(ntp_time);
2515 ntp_time += kFrameIntervalMs;
2516 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002517
2518 video_stream_encoder_->Stop();
2519}
2520
mflodmancc3d4422017-08-03 08:27:51 -07002521TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002523 DataRate::BitsPerSec(kTargetBitrateBps),
2524 DataRate::BitsPerSec(kTargetBitrateBps),
2525 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002526 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002527
sprangc5d62e22017-04-02 23:53:04 -07002528 const int kFrameWidth = 1280;
2529 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002530
Åsa Persson8c1bf952018-09-13 10:42:19 +02002531 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002532
kthelgason5e13d412016-12-01 03:59:51 -08002533 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002534 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002535 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002536 frame_timestamp += kFrameIntervalMs;
2537
perkj803d97f2016-11-01 11:45:46 -07002538 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002539 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002540 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002541 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002542 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002543 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002544
asapersson0944a802017-04-07 00:57:58 -07002545 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002546 // wanted resolution.
2547 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2548 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2549 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002550 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002551
2552 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002553 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002554 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002555 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002556 // Give the encoder queue time to process the change in degradation preference
2557 // by waiting for an encoded frame.
2558 new_video_source.IncomingCapturedFrame(
2559 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2560 sink_.WaitForEncodedFrame(frame_timestamp);
2561 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002562 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002563 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002564
sprangc5d62e22017-04-02 23:53:04 -07002565 // Force an input frame rate to be available, or the adaptation call won't
2566 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002567 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002568 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002569 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002570 stats_proxy_->SetMockStats(stats);
2571
mflodmancc3d4422017-08-03 08:27:51 -07002572 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002573 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002574 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002575 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002576 frame_timestamp += kFrameIntervalMs;
2577
2578 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002579 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002580 EXPECT_EQ(std::numeric_limits<int>::max(),
2581 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002582 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002583
asapersson02465b82017-04-10 01:12:52 -07002584 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002585 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2586 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002587 // Give the encoder queue time to process the change in degradation preference
2588 // by waiting for an encoded frame.
2589 new_video_source.IncomingCapturedFrame(
2590 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2591 sink_.WaitForEncodedFrame(frame_timestamp);
2592 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002593 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002594
mflodmancc3d4422017-08-03 08:27:51 -07002595 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002596 new_video_source.IncomingCapturedFrame(
2597 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002598 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002599 frame_timestamp += kFrameIntervalMs;
2600
2601 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002602 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002603
2604 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002605 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002606 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002607 // Give the encoder queue time to process the change in degradation preference
2608 // by waiting for an encoded frame.
2609 new_video_source.IncomingCapturedFrame(
2610 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2611 sink_.WaitForEncodedFrame(frame_timestamp);
2612 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002613 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2614 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002615 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002616 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002617
2618 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002619 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002620 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002621 // Give the encoder queue time to process the change in degradation preference
2622 // by waiting for an encoded frame.
2623 new_video_source.IncomingCapturedFrame(
2624 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2625 sink_.WaitForEncodedFrame(frame_timestamp);
2626 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002627 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2628 EXPECT_EQ(std::numeric_limits<int>::max(),
2629 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002630 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002631
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002633}
2634
mflodmancc3d4422017-08-03 08:27:51 -07002635TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002637 DataRate::BitsPerSec(kTargetBitrateBps),
2638 DataRate::BitsPerSec(kTargetBitrateBps),
2639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002640
asaperssonfab67072017-04-04 05:51:49 -07002641 const int kWidth = 1280;
2642 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002643 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002644 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002645 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2646 EXPECT_FALSE(stats.bw_limited_resolution);
2647 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2648
2649 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002650 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002651 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002652 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002653
2654 stats = stats_proxy_->GetStats();
2655 EXPECT_TRUE(stats.bw_limited_resolution);
2656 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2657
2658 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002659 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002660 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002661 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002662
2663 stats = stats_proxy_->GetStats();
2664 EXPECT_FALSE(stats.bw_limited_resolution);
2665 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2666 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2667
mflodmancc3d4422017-08-03 08:27:51 -07002668 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002669}
2670
mflodmancc3d4422017-08-03 08:27:51 -07002671TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002673 DataRate::BitsPerSec(kTargetBitrateBps),
2674 DataRate::BitsPerSec(kTargetBitrateBps),
2675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002676
2677 const int kWidth = 1280;
2678 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002679 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002680 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002681 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2682 EXPECT_FALSE(stats.cpu_limited_resolution);
2683 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2684
2685 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002686 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002687 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002688 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002689
2690 stats = stats_proxy_->GetStats();
2691 EXPECT_TRUE(stats.cpu_limited_resolution);
2692 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2693
2694 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002695 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002696 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002697 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002698
2699 stats = stats_proxy_->GetStats();
2700 EXPECT_FALSE(stats.cpu_limited_resolution);
2701 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002702 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002703
mflodmancc3d4422017-08-03 08:27:51 -07002704 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002705}
2706
mflodmancc3d4422017-08-03 08:27:51 -07002707TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002709 DataRate::BitsPerSec(kTargetBitrateBps),
2710 DataRate::BitsPerSec(kTargetBitrateBps),
2711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002712
asaperssonfab67072017-04-04 05:51:49 -07002713 const int kWidth = 1280;
2714 const int kHeight = 720;
2715 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002716 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002717 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002718 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002719 EXPECT_FALSE(stats.cpu_limited_resolution);
2720 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2721
asaperssonfab67072017-04-04 05:51:49 -07002722 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002723 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002724 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002725 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002726 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002727 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002728 EXPECT_TRUE(stats.cpu_limited_resolution);
2729 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2730
2731 // Set new source with adaptation still enabled.
2732 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002734 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002735
asaperssonfab67072017-04-04 05:51:49 -07002736 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002737 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002738 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002739 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002740 EXPECT_TRUE(stats.cpu_limited_resolution);
2741 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2742
2743 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002744 video_stream_encoder_->SetSource(&new_video_source,
2745 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002746
asaperssonfab67072017-04-04 05:51:49 -07002747 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002748 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002749 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002750 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002751 EXPECT_FALSE(stats.cpu_limited_resolution);
2752 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2753
2754 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002755 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002756 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002757
asaperssonfab67072017-04-04 05:51:49 -07002758 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002759 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002760 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002761 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002762 EXPECT_TRUE(stats.cpu_limited_resolution);
2763 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2764
asaperssonfab67072017-04-04 05:51:49 -07002765 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002766 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002767 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002768 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002769 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002770 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002771 EXPECT_FALSE(stats.cpu_limited_resolution);
2772 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002773 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002774
mflodmancc3d4422017-08-03 08:27:51 -07002775 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002776}
2777
mflodmancc3d4422017-08-03 08:27:51 -07002778TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002780 DataRate::BitsPerSec(kTargetBitrateBps),
2781 DataRate::BitsPerSec(kTargetBitrateBps),
2782 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002783
asaperssonfab67072017-04-04 05:51:49 -07002784 const int kWidth = 1280;
2785 const int kHeight = 720;
2786 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002787 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002788 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002789 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002790 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002791 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002792
2793 // Set new source with adaptation still enabled.
2794 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002795 video_stream_encoder_->SetSource(&new_video_source,
2796 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002797
asaperssonfab67072017-04-04 05:51:49 -07002798 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002799 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002800 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002801 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002802 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002803 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002804
asaperssonfab67072017-04-04 05:51:49 -07002805 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002807 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002808 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002809 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002810 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002811 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002812 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002813
asaperssonfab67072017-04-04 05:51:49 -07002814 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002815 video_stream_encoder_->SetSource(&new_video_source,
2816 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002817
asaperssonfab67072017-04-04 05:51:49 -07002818 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002820 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002821 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002822 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002823 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002824
asapersson02465b82017-04-10 01:12:52 -07002825 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002827 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002828
asaperssonfab67072017-04-04 05:51:49 -07002829 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002830 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002831 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002832 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002833 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002834 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2835 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002836
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002838}
2839
mflodmancc3d4422017-08-03 08:27:51 -07002840TEST_F(VideoStreamEncoderTest,
2841 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002843 DataRate::BitsPerSec(kTargetBitrateBps),
2844 DataRate::BitsPerSec(kTargetBitrateBps),
2845 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002846
2847 const int kWidth = 1280;
2848 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002849 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002850 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002851 video_source_.IncomingCapturedFrame(
2852 CreateFrame(timestamp_ms, kWidth, kHeight));
2853 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2857
2858 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002860 timestamp_ms += kFrameIntervalMs;
2861 video_source_.IncomingCapturedFrame(
2862 CreateFrame(timestamp_ms, kWidth, kHeight));
2863 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002864 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2866 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2867
2868 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002870 timestamp_ms += kFrameIntervalMs;
2871 video_source_.IncomingCapturedFrame(
2872 CreateFrame(timestamp_ms, kWidth, kHeight));
2873 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002874 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2875 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2876 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2877
Niels Möller4db138e2018-04-19 09:04:13 +02002878 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002879 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002880
2881 VideoEncoderConfig video_encoder_config;
2882 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2883 // Make format different, to force recreation of encoder.
2884 video_encoder_config.video_format.parameters["foo"] = "foo";
2885 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002886 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002887 timestamp_ms += kFrameIntervalMs;
2888 video_source_.IncomingCapturedFrame(
2889 CreateFrame(timestamp_ms, kWidth, kHeight));
2890 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002891 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2892 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2894
mflodmancc3d4422017-08-03 08:27:51 -07002895 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002896}
2897
mflodmancc3d4422017-08-03 08:27:51 -07002898TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002899 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002901 DataRate::BitsPerSec(kTargetBitrateBps),
2902 DataRate::BitsPerSec(kTargetBitrateBps),
2903 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2904
2905 const int kWidth = 1280;
2906 const int kHeight = 720;
2907 int sequence = 1;
2908
2909 // Enable BALANCED preference, no initial limitation.
2910 test::FrameForwarder source;
2911 video_stream_encoder_->SetSource(&source,
2912 webrtc::DegradationPreference::BALANCED);
2913 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2914 WaitForEncodedFrame(sequence++);
2915 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2916 EXPECT_FALSE(stats.cpu_limited_resolution);
2917 EXPECT_FALSE(stats.cpu_limited_framerate);
2918 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2919
2920 // Trigger CPU overuse, should now adapt down.
2921 video_stream_encoder_->TriggerCpuOveruse();
2922 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2923 WaitForEncodedFrame(sequence++);
2924 stats = stats_proxy_->GetStats();
2925 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2926
2927 // Set new degradation preference should clear restrictions since we changed
2928 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002929 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002930 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2931 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2932 WaitForEncodedFrame(sequence++);
2933 stats = stats_proxy_->GetStats();
2934 EXPECT_FALSE(stats.cpu_limited_resolution);
2935 EXPECT_FALSE(stats.cpu_limited_framerate);
2936 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2937
2938 // Force an input frame rate to be available, or the adaptation call won't
2939 // know what framerate to adapt from.
2940 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2941 mock_stats.input_frame_rate = 30;
2942 stats_proxy_->SetMockStats(mock_stats);
2943 video_stream_encoder_->TriggerCpuOveruse();
2944 stats_proxy_->ResetMockStats();
2945 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2946 WaitForEncodedFrame(sequence++);
2947
2948 // We have now adapted once.
2949 stats = stats_proxy_->GetStats();
2950 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2951
2952 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002953 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2954 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002955 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2956 WaitForEncodedFrame(sequence++);
2957 stats = stats_proxy_->GetStats();
2958 EXPECT_FALSE(stats.cpu_limited_resolution);
2959 EXPECT_FALSE(stats.cpu_limited_framerate);
2960 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2961
2962 video_stream_encoder_->Stop();
2963}
2964
2965TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002966 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002968 DataRate::BitsPerSec(kTargetBitrateBps),
2969 DataRate::BitsPerSec(kTargetBitrateBps),
2970 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002971
asapersson0944a802017-04-07 00:57:58 -07002972 const int kWidth = 1280;
2973 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002974 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002975
asaperssonfab67072017-04-04 05:51:49 -07002976 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002977 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002978 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002979 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002980 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002981 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2982
asapersson02465b82017-04-10 01:12:52 -07002983 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002985 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002986 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002987 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002988 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002989 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07002990 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2991
2992 // Set new source with adaptation still enabled.
2993 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002994 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002995 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002996
2997 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07002998 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002999 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003000 stats = stats_proxy_->GetStats();
3001 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003002 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003003 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3004
sprangc5d62e22017-04-02 23:53:04 -07003005 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003007 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003008 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003009 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003010 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003011 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003012 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003013 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003014 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003015 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3016
sprangc5d62e22017-04-02 23:53:04 -07003017 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003018 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003019 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3020 mock_stats.input_frame_rate = 30;
3021 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003022 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003023 stats_proxy_->ResetMockStats();
3024
3025 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003026 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003027 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003028
3029 // Framerate now adapted.
3030 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003031 EXPECT_FALSE(stats.cpu_limited_resolution);
3032 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003033 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3034
3035 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003036 video_stream_encoder_->SetSource(&new_video_source,
3037 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003038 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003039 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003040 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003041
3042 stats = stats_proxy_->GetStats();
3043 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003044 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003045 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3046
3047 // Try to trigger overuse. Should not succeed.
3048 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003050 stats_proxy_->ResetMockStats();
3051
3052 stats = stats_proxy_->GetStats();
3053 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003054 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003055 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3056
3057 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003058 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003059 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003060 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003061 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003062 stats = stats_proxy_->GetStats();
3063 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003064 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003065 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003066
3067 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003068 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003069 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003070 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003071 stats = stats_proxy_->GetStats();
3072 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003073 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003074 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3075
3076 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003077 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003078 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003079 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003080 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003081 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003082 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003083 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003084 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003085 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003086 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3087
3088 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003089 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003090 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003091 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003092 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003093 stats = stats_proxy_->GetStats();
3094 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003095 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003096 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003097 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003098
mflodmancc3d4422017-08-03 08:27:51 -07003099 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003100}
3101
mflodmancc3d4422017-08-03 08:27:51 -07003102TEST_F(VideoStreamEncoderTest,
3103 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003104 const int kWidth = 1280;
3105 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003107 DataRate::BitsPerSec(kTargetBitrateBps),
3108 DataRate::BitsPerSec(kTargetBitrateBps),
3109 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003110
asaperssonfab67072017-04-04 05:51:49 -07003111 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003112 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003113
asaperssonfab67072017-04-04 05:51:49 -07003114 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003116
asaperssonfab67072017-04-04 05:51:49 -07003117 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003118 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003119
asaperssonfab67072017-04-04 05:51:49 -07003120 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003121 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003122
kthelgason876222f2016-11-29 01:44:11 -08003123 // Expect a scale down.
3124 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003125 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003126
asapersson02465b82017-04-10 01:12:52 -07003127 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003128 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003129 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003130 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003131
asaperssonfab67072017-04-04 05:51:49 -07003132 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003133 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003134 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003135 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003136
asaperssonfab67072017-04-04 05:51:49 -07003137 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003138 EXPECT_EQ(std::numeric_limits<int>::max(),
3139 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003140
asaperssonfab67072017-04-04 05:51:49 -07003141 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003142 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003143 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003144 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003145
asapersson02465b82017-04-10 01:12:52 -07003146 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003147 EXPECT_EQ(std::numeric_limits<int>::max(),
3148 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003149
mflodmancc3d4422017-08-03 08:27:51 -07003150 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003151}
3152
mflodmancc3d4422017-08-03 08:27:51 -07003153TEST_F(VideoStreamEncoderTest,
3154 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003155 const int kWidth = 1280;
3156 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003157 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003158 DataRate::BitsPerSec(kTargetBitrateBps),
3159 DataRate::BitsPerSec(kTargetBitrateBps),
3160 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003161
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003162 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003163 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003166
3167 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003168 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003169 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003170 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3171 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3172
3173 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003174 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003175 EXPECT_THAT(source.sink_wants(),
3176 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003177 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3178 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3179 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3180
3181 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003182 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003183 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3184 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3185 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3186
mflodmancc3d4422017-08-03 08:27:51 -07003187 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003188}
3189
mflodmancc3d4422017-08-03 08:27:51 -07003190TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003191 const int kWidth = 1280;
3192 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003194 DataRate::BitsPerSec(kTargetBitrateBps),
3195 DataRate::BitsPerSec(kTargetBitrateBps),
3196 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003197
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003198 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003199 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003200 video_stream_encoder_->SetSource(&source,
3201 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003202 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3203 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003204 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003205
3206 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003208 EXPECT_THAT(source.sink_wants(),
3209 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003210 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3211 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3212 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3213
3214 // Trigger adapt down for same input resolution, expect no change.
3215 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3216 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003217 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003218 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3219 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3220 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3221
3222 // Trigger adapt down for larger input resolution, expect no change.
3223 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3224 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003225 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003226 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3227 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3228 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3229
mflodmancc3d4422017-08-03 08:27:51 -07003230 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003231}
3232
mflodmancc3d4422017-08-03 08:27:51 -07003233TEST_F(VideoStreamEncoderTest,
3234 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003235 const int kWidth = 1280;
3236 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003238 DataRate::BitsPerSec(kTargetBitrateBps),
3239 DataRate::BitsPerSec(kTargetBitrateBps),
3240 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003241
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003242 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003243 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003244 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003245 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003246
3247 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003249 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003250 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3251 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3252
3253 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003254 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003255 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003256 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3257 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3258
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003260}
3261
mflodmancc3d4422017-08-03 08:27:51 -07003262TEST_F(VideoStreamEncoderTest,
3263 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003264 const int kWidth = 1280;
3265 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003267 DataRate::BitsPerSec(kTargetBitrateBps),
3268 DataRate::BitsPerSec(kTargetBitrateBps),
3269 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003270
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003271 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003272 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003273 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003274 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003275
3276 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003277 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003278 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003279 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003280 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3281
3282 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003283 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003284 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003285 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003286 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3287
mflodmancc3d4422017-08-03 08:27:51 -07003288 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003289}
3290
mflodmancc3d4422017-08-03 08:27:51 -07003291TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003292 const int kWidth = 1280;
3293 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003294 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003295 DataRate::BitsPerSec(kTargetBitrateBps),
3296 DataRate::BitsPerSec(kTargetBitrateBps),
3297 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003298
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003299 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003300 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003301 video_stream_encoder_->SetSource(&source,
3302 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003303
3304 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3305 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003306 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3308 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3309 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3310
3311 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003312 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003313 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3315 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3316 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3317
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003319}
3320
mflodmancc3d4422017-08-03 08:27:51 -07003321TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003322 const int kWidth = 1280;
3323 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003325 DataRate::BitsPerSec(kTargetBitrateBps),
3326 DataRate::BitsPerSec(kTargetBitrateBps),
3327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003328
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003329 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003330 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 video_stream_encoder_->SetSource(&source,
3332 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003333
3334 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3335 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003336 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003337 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3338 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3339 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3340
3341 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3347
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003349}
3350
mflodmancc3d4422017-08-03 08:27:51 -07003351TEST_F(VideoStreamEncoderTest,
3352 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003353 const int kWidth = 1280;
3354 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003356 DataRate::BitsPerSec(kTargetBitrateBps),
3357 DataRate::BitsPerSec(kTargetBitrateBps),
3358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003359
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003360 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003361 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003362 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003363 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003364 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003365
3366 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003367 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003368 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003369 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3371
3372 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003374 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003375 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003376 EXPECT_THAT(source.sink_wants(),
3377 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003378 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3379 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3380
3381 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003383 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3385 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3387
mflodmancc3d4422017-08-03 08:27:51 -07003388 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003389}
3390
mflodmancc3d4422017-08-03 08:27:51 -07003391TEST_F(VideoStreamEncoderTest,
3392 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003393 const int kWidth = 1280;
3394 const int kHeight = 720;
3395 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003397 DataRate::BitsPerSec(kTargetBitrateBps),
3398 DataRate::BitsPerSec(kTargetBitrateBps),
3399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003400
3401 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3402 stats.input_frame_rate = kInputFps;
3403 stats_proxy_->SetMockStats(stats);
3404
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003405 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003406 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3407 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003408 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003409
3410 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003411 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003412 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3413 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003414 EXPECT_THAT(video_source_.sink_wants(),
3415 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003416
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003417 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003418 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003419 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003420 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003421 // Give the encoder queue time to process the change in degradation preference
3422 // by waiting for an encoded frame.
3423 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003425 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003426
3427 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003428 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003429 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3430 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003431 EXPECT_THAT(new_video_source.sink_wants(),
3432 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003433
3434 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003435 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003436 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003437
mflodmancc3d4422017-08-03 08:27:51 -07003438 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003439}
3440
mflodmancc3d4422017-08-03 08:27:51 -07003441TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003442 const int kWidth = 1280;
3443 const int kHeight = 720;
3444 const size_t kNumFrames = 10;
3445
Henrik Boström381d1092020-05-12 18:49:07 +02003446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003447 DataRate::BitsPerSec(kTargetBitrateBps),
3448 DataRate::BitsPerSec(kTargetBitrateBps),
3449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003450
asaperssond0de2952017-04-21 01:47:31 -07003451 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003452 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003453 video_source_.set_adaptation_enabled(true);
3454
3455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3457
3458 int downscales = 0;
3459 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003460 video_source_.IncomingCapturedFrame(
3461 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3462 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003463
asaperssonfab67072017-04-04 05:51:49 -07003464 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003465 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003466 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003467 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003468
3469 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3470 ++downscales;
3471
3472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3473 EXPECT_EQ(downscales,
3474 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003476 }
mflodmancc3d4422017-08-03 08:27:51 -07003477 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003478}
3479
mflodmancc3d4422017-08-03 08:27:51 -07003480TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003481 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3482 const int kWidth = 1280;
3483 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003485 DataRate::BitsPerSec(kTargetBitrateBps),
3486 DataRate::BitsPerSec(kTargetBitrateBps),
3487 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003488
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003489 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003490 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003491 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003492 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003493 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003494
Åsa Persson8c1bf952018-09-13 10:42:19 +02003495 int64_t timestamp_ms = kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003497 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003499 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3500 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3501
3502 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003503 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003504 timestamp_ms += kFrameIntervalMs;
3505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3506 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003507 EXPECT_THAT(source.sink_wants(),
3508 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003509 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3510 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3511
3512 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003513 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003514 timestamp_ms += kFrameIntervalMs;
3515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003516 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003518 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3519 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3520
3521 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003522 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003523 timestamp_ms += kFrameIntervalMs;
3524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3525 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003526 EXPECT_THAT(source.sink_wants(),
3527 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003528 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3529 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3530
3531 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003532 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003533 timestamp_ms += kFrameIntervalMs;
3534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003535 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003536 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003537 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3539
mflodmancc3d4422017-08-03 08:27:51 -07003540 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003541}
3542
mflodmancc3d4422017-08-03 08:27:51 -07003543TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003544 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3545 const int kWidth = 1280;
3546 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003548 DataRate::BitsPerSec(kTargetBitrateBps),
3549 DataRate::BitsPerSec(kTargetBitrateBps),
3550 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003551
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003552 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003553 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003554 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003555 video_stream_encoder_->SetSource(&source,
3556 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003557
Åsa Persson8c1bf952018-09-13 10:42:19 +02003558 int64_t timestamp_ms = kFrameIntervalMs;
3559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003560 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003561 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003566 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003570 EXPECT_THAT(source.sink_wants(),
3571 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3573 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3574
3575 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003576 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003577 timestamp_ms += kFrameIntervalMs;
3578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003579 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003580 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3582 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3583
3584 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003585 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003586 timestamp_ms += kFrameIntervalMs;
3587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3588 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003589 EXPECT_THAT(source.sink_wants(),
3590 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3592 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3593
3594 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003595 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003596 timestamp_ms += kFrameIntervalMs;
3597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003598 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003599 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3601 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602
mflodmancc3d4422017-08-03 08:27:51 -07003603 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003604}
3605
Sergey Silkin41c650b2019-10-14 13:12:19 +02003606TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3607 fake_encoder_.SetResolutionBitrateLimits(
3608 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3609
Henrik Boström381d1092020-05-12 18:49:07 +02003610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003611 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3612 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3613 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3614 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003615
3616 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003617 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003618 source.set_adaptation_enabled(true);
3619 video_stream_encoder_->SetSource(
3620 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3621
3622 // Insert 720p frame.
3623 int64_t timestamp_ms = kFrameIntervalMs;
3624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3625 WaitForEncodedFrame(1280, 720);
3626
3627 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003629 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3630 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3631 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3632 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003633 video_stream_encoder_->TriggerQualityLow();
3634
3635 // Insert 720p frame. It should be downscaled and encoded.
3636 timestamp_ms += kFrameIntervalMs;
3637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3638 WaitForEncodedFrame(960, 540);
3639
3640 // Trigger adapt up. Higher resolution should not be requested duo to lack
3641 // of bitrate.
3642 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003643 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003644
3645 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003647 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3648 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3649 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3650 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003651
3652 // Trigger adapt up. Higher resolution should be requested.
3653 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003654 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003655
3656 video_stream_encoder_->Stop();
3657}
3658
3659TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3660 fake_encoder_.SetResolutionBitrateLimits(
3661 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3662
3663 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003665 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3666 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3667 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3668 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003669
3670 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003671 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003672 source.set_adaptation_enabled(true);
3673 video_stream_encoder_->SetSource(
3674 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3675
3676 // Insert 720p frame. It should be dropped and lower resolution should be
3677 // requested.
3678 int64_t timestamp_ms = kFrameIntervalMs;
3679 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3680 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003681 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003682
3683 // Insert 720p frame. It should be downscaled and encoded.
3684 timestamp_ms += kFrameIntervalMs;
3685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3686 WaitForEncodedFrame(960, 540);
3687
3688 video_stream_encoder_->Stop();
3689}
3690
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003691class BalancedDegradationTest : public VideoStreamEncoderTest {
3692 protected:
3693 void SetupTest() {
3694 // Reset encoder for field trials to take effect.
3695 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003696 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003697
3698 // Enable BALANCED preference.
3699 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003700 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3701 }
3702
3703 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003705 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3706 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003707 }
3708
Åsa Persson45b176f2019-09-30 11:19:05 +02003709 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003710 timestamp_ms_ += kFrameIntervalMs;
3711 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003712 }
3713
3714 void InsertFrameAndWaitForEncoded() {
3715 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003716 sink_.WaitForEncodedFrame(timestamp_ms_);
3717 }
3718
3719 const int kWidth = 640; // pixels:640x360=230400
3720 const int kHeight = 360;
3721 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3722 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003723 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003724};
3725
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003726TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003727 test::ScopedFieldTrials field_trials(
3728 "WebRTC-Video-BalancedDegradationSettings/"
3729 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3730 SetupTest();
3731
3732 // Force input frame rate.
3733 const int kInputFps = 24;
3734 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3735 stats.input_frame_rate = kInputFps;
3736 stats_proxy_->SetMockStats(stats);
3737
Åsa Persson45b176f2019-09-30 11:19:05 +02003738 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003739 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003740
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003741 // Trigger adapt down, expect scaled down framerate and resolution,
3742 // since Fps diff (input-requested:0) < threshold.
3743 video_stream_encoder_->TriggerQualityLow();
3744 EXPECT_THAT(source_.sink_wants(),
3745 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003746
3747 video_stream_encoder_->Stop();
3748}
3749
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003750TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003751 test::ScopedFieldTrials field_trials(
3752 "WebRTC-Video-BalancedDegradationSettings/"
3753 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3754 SetupTest();
3755
3756 // Force input frame rate.
3757 const int kInputFps = 25;
3758 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3759 stats.input_frame_rate = kInputFps;
3760 stats_proxy_->SetMockStats(stats);
3761
Åsa Persson45b176f2019-09-30 11:19:05 +02003762 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003763 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003764
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003765 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3766 // Fps diff (input-requested:1) == threshold.
3767 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003768 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003769
3770 video_stream_encoder_->Stop();
3771}
3772
3773TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3774 test::ScopedFieldTrials field_trials(
3775 "WebRTC-Video-BalancedDegradationSettings/"
3776 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3777 SetupTest();
3778
3779 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3780
Åsa Persson45b176f2019-09-30 11:19:05 +02003781 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003782 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003783
3784 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3785 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003786 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003787
3788 video_stream_encoder_->Stop();
3789}
3790
Åsa Perssonccfb3402019-09-25 15:13:04 +02003791TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003792 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003793 "WebRTC-Video-BalancedDegradationSettings/"
3794 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003795 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003796
Åsa Persson1b247f12019-08-14 17:26:39 +02003797 const int kMinBitrateBps = 425000;
3798 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003799 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003800
Åsa Persson45b176f2019-09-30 11:19:05 +02003801 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003802 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003803 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3804
3805 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3806 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003807 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003808 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003809 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3810
3811 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3812 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003813 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003814 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003815 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3816
Åsa Persson30ab0152019-08-27 12:22:33 +02003817 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3818 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003819 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003820 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003821 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003822 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823
3824 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003825 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003826 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003827 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003828
Åsa Persson30ab0152019-08-27 12:22:33 +02003829 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003830 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003831 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003832 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003833 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003834 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3835
3836 video_stream_encoder_->Stop();
3837}
3838
Åsa Perssonccfb3402019-09-25 15:13:04 +02003839TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003840 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3841 test::ScopedFieldTrials field_trials(
3842 "WebRTC-Video-BalancedDegradationSettings/"
3843 "pixels:57600|129600|230400,fps:7|24|24/");
3844 SetupTest();
3845 OnBitrateUpdated(kLowTargetBitrateBps);
3846
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003847 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003848
3849 // Insert frame, expect scaled down:
3850 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3851 InsertFrame();
3852 EXPECT_FALSE(WaitForFrame(1000));
3853 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3854 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3855
3856 // Insert frame, expect scaled down:
3857 // resolution (320x180@24fps).
3858 InsertFrame();
3859 EXPECT_FALSE(WaitForFrame(1000));
3860 EXPECT_LT(source_.sink_wants().max_pixel_count,
3861 source_.last_wants().max_pixel_count);
3862 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3863
3864 // Frame should not be dropped (min pixels per frame reached).
3865 InsertFrameAndWaitForEncoded();
3866
3867 video_stream_encoder_->Stop();
3868}
3869
3870TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003871 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003872 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003873 "WebRTC-Video-BalancedDegradationSettings/"
3874 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003875 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003876
Åsa Persson30ab0152019-08-27 12:22:33 +02003877 const int kResolutionMinBitrateBps = 435000;
3878 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003879 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003880
Åsa Persson45b176f2019-09-30 11:19:05 +02003881 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003882 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003883 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3884
3885 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3886 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003887 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003888 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003889 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3890
3891 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3892 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003893 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003894 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003895 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3896
3897 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3898 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003899 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003900 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003901 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3902
Åsa Persson30ab0152019-08-27 12:22:33 +02003903 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3904 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003905 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003906 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003907 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3908
3909 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3910 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003911 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003912 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913
3914 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003915 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003916 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003917 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003918 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003919 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3920
3921 video_stream_encoder_->Stop();
3922}
3923
Åsa Perssonccfb3402019-09-25 15:13:04 +02003924TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003925 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003926 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003927 "WebRTC-Video-BalancedDegradationSettings/"
3928 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003929 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003930
Åsa Persson30ab0152019-08-27 12:22:33 +02003931 const int kMinBitrateBps = 425000;
3932 const int kTooLowMinBitrateBps = 424000;
3933 const int kResolutionMinBitrateBps = 435000;
3934 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003935 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003936
Åsa Persson45b176f2019-09-30 11:19:05 +02003937 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003939 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3940
3941 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3942 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003943 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003944 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003945 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3946
3947 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003949 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003950 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003951 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3952
3953 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3954 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003955 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003956 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003957 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3958
3959 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3960 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003961 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003962 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3963
3964 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003965 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003966 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003967 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003968 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003969 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
3971 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003972 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003973 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003974 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003975 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3976
3977 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003978 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003979 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003980 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003981 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003982 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3983
Åsa Persson1b247f12019-08-14 17:26:39 +02003984 video_stream_encoder_->Stop();
3985}
3986
mflodmancc3d4422017-08-03 08:27:51 -07003987TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003988 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
3989 const int kWidth = 1280;
3990 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003992 DataRate::BitsPerSec(kTargetBitrateBps),
3993 DataRate::BitsPerSec(kTargetBitrateBps),
3994 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003995
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003996 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003997 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003998 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003999 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004000 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004001
Åsa Persson8c1bf952018-09-13 10:42:19 +02004002 int64_t timestamp_ms = kFrameIntervalMs;
4003 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004004 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004005 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004006 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4007 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4008 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4009 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4010
4011 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004012 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004013 timestamp_ms += kFrameIntervalMs;
4014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4015 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004016 EXPECT_THAT(source.sink_wants(),
4017 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004018 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4019 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4020 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4021 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
4023 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004024 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004025 timestamp_ms += kFrameIntervalMs;
4026 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4027 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004028 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004029 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4030 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4031 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4033
Jonathan Yubc771b72017-12-08 17:04:29 -08004034 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004036 timestamp_ms += kFrameIntervalMs;
4037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4038 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004039 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004042 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4044
Jonathan Yubc771b72017-12-08 17:04:29 -08004045 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004047 timestamp_ms += kFrameIntervalMs;
4048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4049 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004051 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004052 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4053 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4054 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4055 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4056
Jonathan Yubc771b72017-12-08 17:04:29 -08004057 // Trigger quality adapt down, expect no change (min resolution reached).
4058 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004059 timestamp_ms += kFrameIntervalMs;
4060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4061 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004062 EXPECT_THAT(source.sink_wants(), FpsMax());
4063 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004064 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4065 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4066 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4067 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4068
Evan Shrubsole64469032020-06-11 10:45:29 +02004069 // Trigger quality adapt up, expect upscaled resolution (480x270).
4070 video_stream_encoder_->TriggerQualityHigh();
4071 timestamp_ms += kFrameIntervalMs;
4072 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4073 WaitForEncodedFrame(timestamp_ms);
4074 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4077 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4079
4080 // Trigger quality and cpu adapt up since both are most limited, expect
4081 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004082 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004083 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004084 timestamp_ms += kFrameIntervalMs;
4085 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4086 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004087 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004088 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4089 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4090 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004091 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004092
Evan Shrubsole64469032020-06-11 10:45:29 +02004093 // Trigger quality and cpu adapt up since both are most limited, expect
4094 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004095 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004096 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004097 timestamp_ms += kFrameIntervalMs;
4098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4099 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004100 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004101 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004102 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004103 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004104 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4105 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004106
Evan Shrubsole64469032020-06-11 10:45:29 +02004107 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4108 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004109 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004110 timestamp_ms += kFrameIntervalMs;
4111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4112 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004113 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004114 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4115 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004116 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004117 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004118
4119 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004120 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004121 timestamp_ms += kFrameIntervalMs;
4122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004123 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004124 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004125 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004126 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4127 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004128 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004129 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004130
mflodmancc3d4422017-08-03 08:27:51 -07004131 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004132}
4133
mflodmancc3d4422017-08-03 08:27:51 -07004134TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004135 const int kWidth = 640;
4136 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004137
Henrik Boström381d1092020-05-12 18:49:07 +02004138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004139 DataRate::BitsPerSec(kTargetBitrateBps),
4140 DataRate::BitsPerSec(kTargetBitrateBps),
4141 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004142
perkj803d97f2016-11-01 11:45:46 -07004143 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004144 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004145 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004146 }
4147
mflodmancc3d4422017-08-03 08:27:51 -07004148 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004149 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004150 video_source_.IncomingCapturedFrame(CreateFrame(
4151 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004152 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004153 }
4154
mflodmancc3d4422017-08-03 08:27:51 -07004155 video_stream_encoder_->Stop();
4156 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004157 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004158
Ying Wangef3998f2019-12-09 13:06:53 +01004159 EXPECT_METRIC_EQ(
4160 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4161 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004162 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4163}
4164
mflodmancc3d4422017-08-03 08:27:51 -07004165TEST_F(VideoStreamEncoderTest,
4166 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004168 DataRate::BitsPerSec(kTargetBitrateBps),
4169 DataRate::BitsPerSec(kTargetBitrateBps),
4170 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004171 const int kWidth = 640;
4172 const int kHeight = 360;
4173
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004174 video_stream_encoder_->SetSource(&video_source_,
4175 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004176
4177 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4178 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004179 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004180 }
4181
mflodmancc3d4422017-08-03 08:27:51 -07004182 video_stream_encoder_->Stop();
4183 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004184 stats_proxy_.reset();
4185
4186 EXPECT_EQ(0,
4187 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4188}
4189
Per Kjellanderdcef6412020-10-07 15:09:05 +02004190TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4191 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004192 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004193 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004194
4195 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004196 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004197 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004198 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4199 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004200
Henrik Boström381d1092020-05-12 18:49:07 +02004201 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004202 DataRate::BitsPerSec(kLowTargetBitrateBps),
4203 DataRate::BitsPerSec(kLowTargetBitrateBps),
4204 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004205
sprang57c2fff2017-01-16 06:24:02 -08004206 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004207 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4208 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004209 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4210 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4211
Erik Språngd7329ca2019-02-21 21:19:53 +01004212 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004213 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004214 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004215
Per Kjellanderdcef6412020-10-07 15:09:05 +02004216 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004217 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004218 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4219 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004220 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004221 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004222
Per Kjellanderdcef6412020-10-07 15:09:05 +02004223 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004224 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004225 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004226 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004227 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4228 WaitForEncodedFrame(CurrentTimeMs());
4229 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004230 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004231 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004232
mflodmancc3d4422017-08-03 08:27:51 -07004233 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004234}
4235
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004236TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004237 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004238 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004239 kVideoLayersAllocation);
4240
4241 const int kDefaultFps = 30;
4242
4243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4244 DataRate::BitsPerSec(kLowTargetBitrateBps),
4245 DataRate::BitsPerSec(kLowTargetBitrateBps),
4246 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4247
4248 video_source_.IncomingCapturedFrame(
4249 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4250 WaitForEncodedFrame(CurrentTimeMs());
4251 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4252 VideoLayersAllocation last_layer_allocation =
4253 sink_.GetLastVideoLayersAllocation();
4254 // kLowTargetBitrateBps is only enough for one spatial layer.
4255 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4256
4257 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004258 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004259 // Check that encoder has been updated too, not just allocation observer.
4260 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4261 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4262
Erik Språng9d69cbe2020-10-22 17:44:42 +02004263 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004264 int number_of_layers_allocation = 1;
4265 const int64_t start_time_ms = CurrentTimeMs();
4266 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4267 video_source_.IncomingCapturedFrame(
4268 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4269 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004270 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4271 number_of_layers_allocation = sink_.number_of_layers_allocations();
4272 VideoLayersAllocation new_allocation =
4273 sink_.GetLastVideoLayersAllocation();
4274 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4275 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4276 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4277 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4278 .target_bitrate_per_temporal_layer,
4279 last_layer_allocation.active_spatial_layers[0]
4280 .target_bitrate_per_temporal_layer);
4281 last_layer_allocation = new_allocation;
4282 }
4283 }
4284 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4285 video_stream_encoder_->Stop();
4286}
4287
4288TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004289 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004290 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4291 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4292 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004293 VideoEncoderConfig video_encoder_config;
4294 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4295 /* num_streams*/ 3, &video_encoder_config);
4296 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4297 video_encoder_config.content_type =
4298 VideoEncoderConfig::ContentType::kRealtimeVideo;
4299 video_encoder_config.encoder_specific_settings =
4300 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4301 VideoEncoder::GetDefaultVp8Settings());
4302 for (auto& layer : video_encoder_config.simulcast_layers) {
4303 layer.num_temporal_layers = 2;
4304 }
4305 // Simulcast layers are used for enabling/disabling streams.
4306 video_encoder_config.simulcast_layers[0].active = true;
4307 video_encoder_config.simulcast_layers[1].active = false;
4308 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004309 ConfigureEncoder(std::move(video_encoder_config),
4310 VideoStreamEncoder::BitrateAllocationCallbackType::
4311 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004312
4313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4314 DataRate::BitsPerSec(kTargetBitrateBps),
4315 DataRate::BitsPerSec(kTargetBitrateBps),
4316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4317
4318 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4319 WaitForEncodedFrame(CurrentTimeMs());
4320 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4321 VideoLayersAllocation last_layer_allocation =
4322 sink_.GetLastVideoLayersAllocation();
4323
4324 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4325 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4326 .target_bitrate_per_temporal_layer,
4327 SizeIs(2));
4328 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4329 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4330 video_stream_encoder_->Stop();
4331}
4332
4333TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004334 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004335 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4336 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4337 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004338 VideoEncoderConfig video_encoder_config;
4339 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4340 /* num_streams*/ 3, &video_encoder_config);
4341 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4342 video_encoder_config.content_type =
4343 VideoEncoderConfig::ContentType::kRealtimeVideo;
4344 video_encoder_config.encoder_specific_settings =
4345 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4346 VideoEncoder::GetDefaultVp8Settings());
4347 for (auto& layer : video_encoder_config.simulcast_layers) {
4348 layer.num_temporal_layers = 2;
4349 }
4350 // Simulcast layers are used for enabling/disabling streams.
4351 video_encoder_config.simulcast_layers[0].active = true;
4352 video_encoder_config.simulcast_layers[1].active = false;
4353 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004354 ConfigureEncoder(std::move(video_encoder_config),
4355 VideoStreamEncoder::BitrateAllocationCallbackType::
4356 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004357
4358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4359 DataRate::BitsPerSec(kTargetBitrateBps),
4360 DataRate::BitsPerSec(kTargetBitrateBps),
4361 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4362
4363 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4364 WaitForEncodedFrame(CurrentTimeMs());
4365 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4366 VideoLayersAllocation last_layer_allocation =
4367 sink_.GetLastVideoLayersAllocation();
4368
4369 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4370 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4371 .target_bitrate_per_temporal_layer,
4372 SizeIs(2));
4373 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4374
4375 video_stream_encoder_->Stop();
4376}
4377
4378TEST_F(VideoStreamEncoderTest,
4379 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4380 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4381 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004382 VideoEncoderConfig video_encoder_config;
4383 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4384 /* num_streams*/ 1, &video_encoder_config);
4385 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4386 video_encoder_config.content_type =
4387 VideoEncoderConfig::ContentType::kRealtimeVideo;
4388 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4389 vp9_settings.numberOfSpatialLayers = 2;
4390 vp9_settings.numberOfTemporalLayers = 2;
4391 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4392 vp9_settings.automaticResizeOn = false;
4393 video_encoder_config.encoder_specific_settings =
4394 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4395 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004396 ConfigureEncoder(std::move(video_encoder_config),
4397 VideoStreamEncoder::BitrateAllocationCallbackType::
4398 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004399
4400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4401 DataRate::BitsPerSec(kTargetBitrateBps),
4402 DataRate::BitsPerSec(kTargetBitrateBps),
4403 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4404
4405 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4406 WaitForEncodedFrame(CurrentTimeMs());
4407 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4408 VideoLayersAllocation last_layer_allocation =
4409 sink_.GetLastVideoLayersAllocation();
4410
4411 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4412 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4413 .target_bitrate_per_temporal_layer,
4414 SizeIs(2));
4415 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4416 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4417 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4418 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4419 .target_bitrate_per_temporal_layer,
4420 SizeIs(2));
4421 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4422 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4423 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4424
4425 // Since full SVC is used, expect the top layer to utilize the full target
4426 // rate.
4427 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4428 .target_bitrate_per_temporal_layer[1],
4429 DataRate::BitsPerSec(kTargetBitrateBps));
4430 video_stream_encoder_->Stop();
4431}
4432
4433TEST_F(VideoStreamEncoderTest,
4434 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4435 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4436 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004437 VideoEncoderConfig video_encoder_config;
4438 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4439 /* num_streams*/ 1, &video_encoder_config);
4440 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4441 video_encoder_config.content_type =
4442 VideoEncoderConfig::ContentType::kRealtimeVideo;
4443 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4444 vp9_settings.numberOfSpatialLayers = 2;
4445 vp9_settings.numberOfTemporalLayers = 2;
4446 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4447 vp9_settings.automaticResizeOn = false;
4448 video_encoder_config.encoder_specific_settings =
4449 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4450 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004451 ConfigureEncoder(std::move(video_encoder_config),
4452 VideoStreamEncoder::BitrateAllocationCallbackType::
4453 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004454
4455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4456 DataRate::BitsPerSec(kTargetBitrateBps),
4457 DataRate::BitsPerSec(kTargetBitrateBps),
4458 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4459
4460 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4461 WaitForEncodedFrame(CurrentTimeMs());
4462 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4463 VideoLayersAllocation last_layer_allocation =
4464 sink_.GetLastVideoLayersAllocation();
4465
4466 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4467 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4468 .target_bitrate_per_temporal_layer,
4469 SizeIs(1));
4470 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4471 .target_bitrate_per_temporal_layer,
4472 SizeIs(1));
4473 // Since full SVC is used, expect the top layer to utilize the full target
4474 // rate.
4475 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4476 .target_bitrate_per_temporal_layer[0],
4477 DataRate::BitsPerSec(kTargetBitrateBps));
4478 video_stream_encoder_->Stop();
4479}
4480
4481TEST_F(VideoStreamEncoderTest,
4482 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4483 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4484 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004485 VideoEncoderConfig video_encoder_config;
4486 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4487 /* num_streams*/ 1, &video_encoder_config);
4488 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4489 video_encoder_config.content_type =
4490 VideoEncoderConfig::ContentType::kRealtimeVideo;
4491 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4492 vp9_settings.numberOfSpatialLayers = 2;
4493 vp9_settings.numberOfTemporalLayers = 2;
4494 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4495 vp9_settings.automaticResizeOn = false;
4496 video_encoder_config.encoder_specific_settings =
4497 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4498 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004499 ConfigureEncoder(std::move(video_encoder_config),
4500 VideoStreamEncoder::BitrateAllocationCallbackType::
4501 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004502
4503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4504 DataRate::BitsPerSec(kTargetBitrateBps),
4505 DataRate::BitsPerSec(kTargetBitrateBps),
4506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4507
4508 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4509 WaitForEncodedFrame(CurrentTimeMs());
4510 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4511 VideoLayersAllocation last_layer_allocation =
4512 sink_.GetLastVideoLayersAllocation();
4513
4514 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4515 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4516 .target_bitrate_per_temporal_layer,
4517 SizeIs(2));
4518 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4519 .target_bitrate_per_temporal_layer,
4520 SizeIs(2));
4521 // Since KSVC is, spatial layers are independend except on key frames.
4522 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4523 .target_bitrate_per_temporal_layer[1],
4524 DataRate::BitsPerSec(kTargetBitrateBps));
4525 video_stream_encoder_->Stop();
4526}
4527
4528TEST_F(VideoStreamEncoderTest,
4529 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4530 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4531 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4532 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004533 VideoEncoderConfig video_encoder_config;
4534 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4535 /* num_streams*/ 1, &video_encoder_config);
4536 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4537 video_encoder_config.content_type =
4538 VideoEncoderConfig::ContentType::kRealtimeVideo;
4539 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4540 vp9_settings.numberOfSpatialLayers = 3;
4541 vp9_settings.numberOfTemporalLayers = 2;
4542 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4543 vp9_settings.automaticResizeOn = false;
4544 video_encoder_config.encoder_specific_settings =
4545 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4546 vp9_settings);
4547 // Simulcast layers are used for enabling/disabling streams.
4548 video_encoder_config.simulcast_layers.resize(3);
4549 video_encoder_config.simulcast_layers[0].active = false;
4550 video_encoder_config.simulcast_layers[1].active = true;
4551 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004552 ConfigureEncoder(std::move(video_encoder_config),
4553 VideoStreamEncoder::BitrateAllocationCallbackType::
4554 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004555
4556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4557 DataRate::BitsPerSec(kTargetBitrateBps),
4558 DataRate::BitsPerSec(kTargetBitrateBps),
4559 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4560
4561 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4562 WaitForEncodedFrame(CurrentTimeMs());
4563 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4564 VideoLayersAllocation last_layer_allocation =
4565 sink_.GetLastVideoLayersAllocation();
4566
4567 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4568 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4569 .target_bitrate_per_temporal_layer,
4570 SizeIs(2));
4571 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4572 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4573
4574 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4575 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4576 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4577 .target_bitrate_per_temporal_layer,
4578 SizeIs(2));
4579 // Since full SVC is used, expect the top layer to utilize the full target
4580 // rate.
4581 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4582 .target_bitrate_per_temporal_layer[1],
4583 DataRate::BitsPerSec(kTargetBitrateBps));
4584 video_stream_encoder_->Stop();
4585}
4586
4587TEST_F(VideoStreamEncoderTest,
4588 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4589 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4590 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4591 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004592 VideoEncoderConfig video_encoder_config;
4593 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4594 /* num_streams*/ 1, &video_encoder_config);
4595 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4596 video_encoder_config.content_type =
4597 VideoEncoderConfig::ContentType::kRealtimeVideo;
4598 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4599 vp9_settings.numberOfSpatialLayers = 3;
4600 vp9_settings.numberOfTemporalLayers = 2;
4601 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4602 vp9_settings.automaticResizeOn = false;
4603 video_encoder_config.encoder_specific_settings =
4604 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4605 vp9_settings);
4606 // Simulcast layers are used for enabling/disabling streams.
4607 video_encoder_config.simulcast_layers.resize(3);
4608 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004609 ConfigureEncoder(std::move(video_encoder_config),
4610 VideoStreamEncoder::BitrateAllocationCallbackType::
4611 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004612
4613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4614 DataRate::BitsPerSec(kTargetBitrateBps),
4615 DataRate::BitsPerSec(kTargetBitrateBps),
4616 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4617
4618 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4619 WaitForEncodedFrame(CurrentTimeMs());
4620 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4621 VideoLayersAllocation last_layer_allocation =
4622 sink_.GetLastVideoLayersAllocation();
4623
4624 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4625 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4626 .target_bitrate_per_temporal_layer,
4627 SizeIs(2));
4628 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4629 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4630
4631 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4632 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4633 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4634 .target_bitrate_per_temporal_layer,
4635 SizeIs(2));
4636 video_stream_encoder_->Stop();
4637}
4638
4639TEST_F(VideoStreamEncoderTest,
4640 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4641 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4642 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4643 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004644 VideoEncoderConfig video_encoder_config;
4645 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4646 /* num_streams*/ 1, &video_encoder_config);
4647 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4648 video_encoder_config.content_type =
4649 VideoEncoderConfig::ContentType::kRealtimeVideo;
4650 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4651 vp9_settings.numberOfSpatialLayers = 3;
4652 vp9_settings.numberOfTemporalLayers = 2;
4653 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4654 vp9_settings.automaticResizeOn = false;
4655 video_encoder_config.encoder_specific_settings =
4656 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4657 vp9_settings);
4658 // Simulcast layers are used for enabling/disabling streams.
4659 video_encoder_config.simulcast_layers.resize(3);
4660 video_encoder_config.simulcast_layers[0].active = false;
4661 video_encoder_config.simulcast_layers[1].active = false;
4662 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004663 ConfigureEncoder(std::move(video_encoder_config),
4664 VideoStreamEncoder::BitrateAllocationCallbackType::
4665 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666
4667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4668 DataRate::BitsPerSec(kTargetBitrateBps),
4669 DataRate::BitsPerSec(kTargetBitrateBps),
4670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4671
4672 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4673 WaitForEncodedFrame(CurrentTimeMs());
4674 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4675 VideoLayersAllocation last_layer_allocation =
4676 sink_.GetLastVideoLayersAllocation();
4677
4678 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4679 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4680 .target_bitrate_per_temporal_layer,
4681 SizeIs(2));
4682 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4683 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4684 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4685 .target_bitrate_per_temporal_layer[1],
4686 DataRate::BitsPerSec(kTargetBitrateBps));
4687 video_stream_encoder_->Stop();
4688}
4689
4690TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4691 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004692 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 kVideoLayersAllocation);
4694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4695 DataRate::BitsPerSec(kTargetBitrateBps),
4696 DataRate::BitsPerSec(kTargetBitrateBps),
4697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4698
4699 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4700 WaitForEncodedFrame(CurrentTimeMs());
4701 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4702 VideoLayersAllocation last_layer_allocation =
4703 sink_.GetLastVideoLayersAllocation();
4704
4705 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4706 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4707 .target_bitrate_per_temporal_layer,
4708 SizeIs(1));
4709 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4710 .target_bitrate_per_temporal_layer[0],
4711 DataRate::BitsPerSec(kTargetBitrateBps));
4712 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4713 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4714 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4715 video_stream_encoder_->Stop();
4716}
4717
4718TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004719 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4720 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004721 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004722 kVideoLayersAllocation);
4723
4724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4725 DataRate::BitsPerSec(kLowTargetBitrateBps),
4726 DataRate::BitsPerSec(kLowTargetBitrateBps),
4727 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4728
4729 video_source_.IncomingCapturedFrame(
4730 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4731 WaitForEncodedFrame(CurrentTimeMs());
4732 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4733 VideoLayersAllocation last_layer_allocation =
4734 sink_.GetLastVideoLayersAllocation();
4735 // kLowTargetBitrateBps is only enough for one spatial layer.
4736 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4737 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4738 .target_bitrate_per_temporal_layer[0],
4739 DataRate::BitsPerSec(kLowTargetBitrateBps));
4740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4742 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4743 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4744 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4745 video_source_.IncomingCapturedFrame(
4746 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4747 WaitForEncodedFrame(CurrentTimeMs());
4748
4749 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4750 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4751 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4752 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4753 .target_bitrate_per_temporal_layer[0],
4754 DataRate::Zero());
4755
4756 video_stream_encoder_->Stop();
4757}
4758
Per Kjellander4190ce92020-12-15 17:24:55 +01004759TEST_F(VideoStreamEncoderTest,
4760 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4761 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004762 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004763 kVideoLayersAllocation);
4764
4765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4766 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4767 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4768 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4769
4770 video_source_.IncomingCapturedFrame(
4771 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4772 WaitForEncodedFrame(CurrentTimeMs());
4773 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4774 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4775 SizeIs(2));
4776 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4777 codec_width_);
4778 EXPECT_EQ(
4779 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4780 codec_height_);
4781
4782 video_source_.IncomingCapturedFrame(
4783 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4784 WaitForEncodedFrame(CurrentTimeMs());
4785 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4786 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4787 SizeIs(2));
4788 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4789 codec_width_ / 2);
4790 EXPECT_EQ(
4791 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4792 codec_height_ / 2);
4793
4794 video_stream_encoder_->Stop();
4795}
4796
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004797TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4798 // 2 TLs configured, temporal layers supported by encoder.
4799 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004800 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004801 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004802 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004803 fake_encoder_.SetTemporalLayersSupported(0, true);
4804
4805 // Bitrate allocated across temporal layers.
4806 const int kTl0Bps = kTargetBitrateBps *
4807 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004808 kNumTemporalLayers, /*temporal_id*/ 0,
4809 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004810 const int kTl1Bps = kTargetBitrateBps *
4811 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004812 kNumTemporalLayers, /*temporal_id*/ 1,
4813 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004814 VideoBitrateAllocation expected_bitrate;
4815 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4816 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4817
4818 VerifyAllocatedBitrate(expected_bitrate);
4819 video_stream_encoder_->Stop();
4820}
4821
4822TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4823 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004824 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004825 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004826 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004827 fake_encoder_.SetTemporalLayersSupported(0, false);
4828
4829 // Temporal layers not supported by the encoder.
4830 // Total bitrate should be at ti:0.
4831 VideoBitrateAllocation expected_bitrate;
4832 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4833
4834 VerifyAllocatedBitrate(expected_bitrate);
4835 video_stream_encoder_->Stop();
4836}
4837
4838TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004839 webrtc::test::ScopedFieldTrials field_trials(
4840 "WebRTC-Video-QualityScalerSettings/"
4841 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4842 // Reset encoder for field trials to take effect.
4843 ConfigureEncoder(video_encoder_config_.Copy());
4844
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004845 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004846 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004847 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004848 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004849 fake_encoder_.SetTemporalLayersSupported(0, true);
4850 fake_encoder_.SetTemporalLayersSupported(1, false);
4851
4852 const int kS0Bps = 150000;
4853 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004854 kS0Bps *
4855 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4856 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004857 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004858 kS0Bps *
4859 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4860 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004861 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4862 // Temporal layers not supported by si:1.
4863 VideoBitrateAllocation expected_bitrate;
4864 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4865 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4866 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4867
4868 VerifyAllocatedBitrate(expected_bitrate);
4869 video_stream_encoder_->Stop();
4870}
4871
Niels Möller7dc26b72017-12-06 10:27:48 +01004872TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4873 const int kFrameWidth = 1280;
4874 const int kFrameHeight = 720;
4875 const int kFramerate = 24;
4876
Henrik Boström381d1092020-05-12 18:49:07 +02004877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004878 DataRate::BitsPerSec(kTargetBitrateBps),
4879 DataRate::BitsPerSec(kTargetBitrateBps),
4880 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004881 test::FrameForwarder source;
4882 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004883 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004884
4885 // Insert a single frame, triggering initial configuration.
4886 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4887 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4888
4889 EXPECT_EQ(
4890 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4891 kDefaultFramerate);
4892
4893 // Trigger reconfigure encoder (without resetting the entire instance).
4894 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004895 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4896 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004897 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004898 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004899 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004900 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4901
4902 // Detector should be updated with fps limit from codec config.
4903 EXPECT_EQ(
4904 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4905 kFramerate);
4906
4907 // Trigger overuse, max framerate should be reduced.
4908 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4909 stats.input_frame_rate = kFramerate;
4910 stats_proxy_->SetMockStats(stats);
4911 video_stream_encoder_->TriggerCpuOveruse();
4912 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4913 int adapted_framerate =
4914 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4915 EXPECT_LT(adapted_framerate, kFramerate);
4916
4917 // Trigger underuse, max framerate should go back to codec configured fps.
4918 // Set extra low fps, to make sure it's actually reset, not just incremented.
4919 stats = stats_proxy_->GetStats();
4920 stats.input_frame_rate = adapted_framerate / 2;
4921 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004922 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004923 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4924 EXPECT_EQ(
4925 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4926 kFramerate);
4927
4928 video_stream_encoder_->Stop();
4929}
4930
4931TEST_F(VideoStreamEncoderTest,
4932 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4933 const int kFrameWidth = 1280;
4934 const int kFrameHeight = 720;
4935 const int kLowFramerate = 15;
4936 const int kHighFramerate = 25;
4937
Henrik Boström381d1092020-05-12 18:49:07 +02004938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004939 DataRate::BitsPerSec(kTargetBitrateBps),
4940 DataRate::BitsPerSec(kTargetBitrateBps),
4941 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004942 test::FrameForwarder source;
4943 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004944 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004945
4946 // Trigger initial configuration.
4947 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004948 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4949 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004950 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004951 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004952 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004953 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004954 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4955
4956 EXPECT_EQ(
4957 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4958 kLowFramerate);
4959
4960 // Trigger overuse, max framerate should be reduced.
4961 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4962 stats.input_frame_rate = kLowFramerate;
4963 stats_proxy_->SetMockStats(stats);
4964 video_stream_encoder_->TriggerCpuOveruse();
4965 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4966 int adapted_framerate =
4967 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4968 EXPECT_LT(adapted_framerate, kLowFramerate);
4969
4970 // Reconfigure the encoder with a new (higher max framerate), max fps should
4971 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004972 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004973 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004975 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977
4978 EXPECT_EQ(
4979 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4980 adapted_framerate);
4981
4982 // Trigger underuse, max framerate should go back to codec configured fps.
4983 stats = stats_proxy_->GetStats();
4984 stats.input_frame_rate = adapted_framerate;
4985 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004986 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004987 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4988 EXPECT_EQ(
4989 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4990 kHighFramerate);
4991
4992 video_stream_encoder_->Stop();
4993}
4994
mflodmancc3d4422017-08-03 08:27:51 -07004995TEST_F(VideoStreamEncoderTest,
4996 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07004997 const int kFrameWidth = 1280;
4998 const int kFrameHeight = 720;
4999 const int kFramerate = 24;
5000
Henrik Boström381d1092020-05-12 18:49:07 +02005001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005002 DataRate::BitsPerSec(kTargetBitrateBps),
5003 DataRate::BitsPerSec(kTargetBitrateBps),
5004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005005 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005006 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005007 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005008
5009 // Trigger initial configuration.
5010 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005011 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5012 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005013 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005014 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005015 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005016 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005018
Niels Möller7dc26b72017-12-06 10:27:48 +01005019 EXPECT_EQ(
5020 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5021 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005022
5023 // Trigger overuse, max framerate should be reduced.
5024 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5025 stats.input_frame_rate = kFramerate;
5026 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005027 video_stream_encoder_->TriggerCpuOveruse();
5028 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005029 int adapted_framerate =
5030 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005031 EXPECT_LT(adapted_framerate, kFramerate);
5032
5033 // Change degradation preference to not enable framerate scaling. Target
5034 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005035 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005037 EXPECT_EQ(
5038 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5039 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005040
mflodmancc3d4422017-08-03 08:27:51 -07005041 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005042}
5043
mflodmancc3d4422017-08-03 08:27:51 -07005044TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005045 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005047 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5048 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5049 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005050 const int kWidth = 640;
5051 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005052
asaperssonfab67072017-04-04 05:51:49 -07005053 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005054
5055 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005056 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005057
5058 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005059 EXPECT_TRUE_WAIT(
5060 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005061
sprangc5d62e22017-04-02 23:53:04 -07005062 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005063
asaperssonfab67072017-04-04 05:51:49 -07005064 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005065 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005066 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005067
5068 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005069 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005070
Henrik Boström2671dac2020-05-19 16:29:09 +02005071 EXPECT_TRUE_WAIT(
5072 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005073
mflodmancc3d4422017-08-03 08:27:51 -07005074 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005075}
5076
mflodmancc3d4422017-08-03 08:27:51 -07005077TEST_F(VideoStreamEncoderTest,
5078 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005079 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005081 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5082 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5083 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005084 const int kWidth = 640;
5085 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005086
5087 // We expect the n initial frames to get dropped.
5088 int i;
5089 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005090 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005091 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005092 }
5093 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005094 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005095 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005096
5097 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005098 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005099
mflodmancc3d4422017-08-03 08:27:51 -07005100 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005101}
5102
mflodmancc3d4422017-08-03 08:27:51 -07005103TEST_F(VideoStreamEncoderTest,
5104 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005105 const int kWidth = 640;
5106 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005108 DataRate::BitsPerSec(kLowTargetBitrateBps),
5109 DataRate::BitsPerSec(kLowTargetBitrateBps),
5110 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005111
5112 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005114 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005115
asaperssonfab67072017-04-04 05:51:49 -07005116 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005117 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005118 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005119
mflodmancc3d4422017-08-03 08:27:51 -07005120 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005121}
5122
mflodmancc3d4422017-08-03 08:27:51 -07005123TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005124 const int kWidth = 640;
5125 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005126 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005127
5128 VideoEncoderConfig video_encoder_config;
5129 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5130 // Make format different, to force recreation of encoder.
5131 video_encoder_config.video_format.parameters["foo"] = "foo";
5132 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005133 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005135 DataRate::BitsPerSec(kLowTargetBitrateBps),
5136 DataRate::BitsPerSec(kLowTargetBitrateBps),
5137 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005138
kthelgasonb83797b2017-02-14 11:57:25 -08005139 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005140 video_stream_encoder_->SetSource(&video_source_,
5141 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005142
asaperssonfab67072017-04-04 05:51:49 -07005143 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005144 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005145 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005146
mflodmancc3d4422017-08-03 08:27:51 -07005147 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005148 fake_encoder_.SetQualityScaling(true);
5149}
5150
Åsa Persson139f4dc2019-08-02 09:29:58 +02005151TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5152 webrtc::test::ScopedFieldTrials field_trials(
5153 "WebRTC-Video-QualityScalerSettings/"
5154 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5155 // Reset encoder for field trials to take effect.
5156 ConfigureEncoder(video_encoder_config_.Copy());
5157 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5158 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5159 const int kWidth = 640;
5160 const int kHeight = 360;
5161
Henrik Boström381d1092020-05-12 18:49:07 +02005162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005163 DataRate::BitsPerSec(kTargetBitrateBps),
5164 DataRate::BitsPerSec(kTargetBitrateBps),
5165 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005166 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5167 // Frame should not be dropped.
5168 WaitForEncodedFrame(1);
5169
Henrik Boström381d1092020-05-12 18:49:07 +02005170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005171 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5172 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5173 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005174 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5175 // Frame should not be dropped.
5176 WaitForEncodedFrame(2);
5177
Henrik Boström381d1092020-05-12 18:49:07 +02005178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005179 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5180 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5181 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005182 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5183 // Expect to drop this frame, the wait should time out.
5184 ExpectDroppedFrame();
5185
5186 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005187 EXPECT_TRUE_WAIT(
5188 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005189 video_stream_encoder_->Stop();
5190}
5191
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005192TEST_F(VideoStreamEncoderTest,
5193 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5194 webrtc::test::ScopedFieldTrials field_trials(
5195 "WebRTC-Video-QualityScalerSettings/"
5196 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5197 fake_encoder_.SetQualityScaling(false);
5198 ConfigureEncoder(video_encoder_config_.Copy());
5199 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5200 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5201 const int kWidth = 640;
5202 const int kHeight = 360;
5203
5204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5205 DataRate::BitsPerSec(kTargetBitrateBps),
5206 DataRate::BitsPerSec(kTargetBitrateBps),
5207 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5208 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5209 // Frame should not be dropped.
5210 WaitForEncodedFrame(1);
5211
5212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5213 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5214 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5215 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5216 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5217 // Frame should not be dropped.
5218 WaitForEncodedFrame(2);
5219
5220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5221 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5222 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5223 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5224 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5225 // Not dropped since quality scaling is disabled.
5226 WaitForEncodedFrame(3);
5227
5228 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005229 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005230 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5231
5232 video_stream_encoder_->Stop();
5233}
5234
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005235TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5236 const int kLowTargetBitrateBps = 400000;
5237 // Set simulcast.
5238 ResetEncoder("VP8", 3, 1, 1, false);
5239 fake_encoder_.SetQualityScaling(true);
5240 const int kWidth = 1280;
5241 const int kHeight = 720;
5242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5243 DataRate::BitsPerSec(kLowTargetBitrateBps),
5244 DataRate::BitsPerSec(kLowTargetBitrateBps),
5245 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5246 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5247 // Frame should not be dropped.
5248 WaitForEncodedFrame(1);
5249
5250 // Trigger QVGA "singlecast"
5251 // Update the config.
5252 VideoEncoderConfig video_encoder_config;
5253 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5254 &video_encoder_config);
5255 for (auto& layer : video_encoder_config.simulcast_layers) {
5256 layer.num_temporal_layers = 1;
5257 layer.max_framerate = kDefaultFramerate;
5258 }
5259 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5260 video_encoder_config.content_type =
5261 VideoEncoderConfig::ContentType::kRealtimeVideo;
5262
5263 video_encoder_config.simulcast_layers[0].active = true;
5264 video_encoder_config.simulcast_layers[1].active = false;
5265 video_encoder_config.simulcast_layers[2].active = false;
5266
5267 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5268 kMaxPayloadLength);
5269 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5270
5271 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5272 // Frame should not be dropped.
5273 WaitForEncodedFrame(2);
5274
5275 // Trigger HD "singlecast"
5276 video_encoder_config.simulcast_layers[0].active = false;
5277 video_encoder_config.simulcast_layers[1].active = false;
5278 video_encoder_config.simulcast_layers[2].active = true;
5279
5280 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5281 kMaxPayloadLength);
5282 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5283
5284 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5285 // Frame should be dropped because of initial frame drop.
5286 ExpectDroppedFrame();
5287
5288 // Expect the sink_wants to specify a scaled frame.
5289 EXPECT_TRUE_WAIT(
5290 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5291 video_stream_encoder_->Stop();
5292}
5293
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005294TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5295 const int kLowTargetBitrateBps = 400000;
5296 // Set simulcast.
5297 ResetEncoder("VP9", 1, 1, 3, false);
5298 fake_encoder_.SetQualityScaling(true);
5299 const int kWidth = 1280;
5300 const int kHeight = 720;
5301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5302 DataRate::BitsPerSec(kLowTargetBitrateBps),
5303 DataRate::BitsPerSec(kLowTargetBitrateBps),
5304 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5305 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5306 // Frame should not be dropped.
5307 WaitForEncodedFrame(1);
5308
5309 // Trigger QVGA "singlecast"
5310 // Update the config.
5311 VideoEncoderConfig video_encoder_config;
5312 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5313 &video_encoder_config);
5314 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5315 vp9_settings.numberOfSpatialLayers = 3;
5316 // Since only one layer is active - automatic resize should be enabled.
5317 vp9_settings.automaticResizeOn = true;
5318 video_encoder_config.encoder_specific_settings =
5319 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5320 vp9_settings);
5321 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5322 video_encoder_config.content_type =
5323 VideoEncoderConfig::ContentType::kRealtimeVideo;
5324 // Currently simulcast layers |active| flags are used to inidicate
5325 // which SVC layers are active.
5326 video_encoder_config.simulcast_layers.resize(3);
5327
5328 video_encoder_config.simulcast_layers[0].active = true;
5329 video_encoder_config.simulcast_layers[1].active = false;
5330 video_encoder_config.simulcast_layers[2].active = false;
5331
5332 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5333 kMaxPayloadLength);
5334 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5335
5336 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5337 // Frame should not be dropped.
5338 WaitForEncodedFrame(2);
5339
5340 // Trigger HD "singlecast"
5341 video_encoder_config.simulcast_layers[0].active = false;
5342 video_encoder_config.simulcast_layers[1].active = false;
5343 video_encoder_config.simulcast_layers[2].active = true;
5344
5345 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5346 kMaxPayloadLength);
5347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5348
5349 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5350 // Frame should be dropped because of initial frame drop.
5351 ExpectDroppedFrame();
5352
5353 // Expect the sink_wants to specify a scaled frame.
5354 EXPECT_TRUE_WAIT(
5355 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5356 video_stream_encoder_->Stop();
5357}
5358
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005359TEST_F(VideoStreamEncoderTest,
5360 InitialFrameDropActivatesWhenResolutionIncreases) {
5361 const int kWidth = 640;
5362 const int kHeight = 360;
5363
5364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5365 DataRate::BitsPerSec(kTargetBitrateBps),
5366 DataRate::BitsPerSec(kTargetBitrateBps),
5367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5368 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5369 // Frame should not be dropped.
5370 WaitForEncodedFrame(1);
5371
5372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5373 DataRate::BitsPerSec(kLowTargetBitrateBps),
5374 DataRate::BitsPerSec(kLowTargetBitrateBps),
5375 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5376 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5377 // Frame should not be dropped, bitrate not too low for frame.
5378 WaitForEncodedFrame(2);
5379
5380 // Incoming resolution increases.
5381 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5382 // Expect to drop this frame, bitrate too low for frame.
5383 ExpectDroppedFrame();
5384
5385 // Expect the sink_wants to specify a scaled frame.
5386 EXPECT_TRUE_WAIT(
5387 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5388 video_stream_encoder_->Stop();
5389}
5390
5391TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5392 const int kWidth = 640;
5393 const int kHeight = 360;
5394 // So that quality scaling doesn't happen by itself.
5395 fake_encoder_.SetQp(kQpHigh);
5396
5397 AdaptingFrameForwarder source(&time_controller_);
5398 source.set_adaptation_enabled(true);
5399 video_stream_encoder_->SetSource(
5400 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5401
5402 int timestamp = 1;
5403
5404 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5405 DataRate::BitsPerSec(kTargetBitrateBps),
5406 DataRate::BitsPerSec(kTargetBitrateBps),
5407 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5408 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5409 WaitForEncodedFrame(timestamp);
5410 timestamp += 9000;
5411 // Long pause to disable all first BWE drop logic.
5412 AdvanceTime(TimeDelta::Millis(1000));
5413
5414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5415 DataRate::BitsPerSec(kLowTargetBitrateBps),
5416 DataRate::BitsPerSec(kLowTargetBitrateBps),
5417 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5418 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5419 // Not dropped frame, as initial frame drop is disabled by now.
5420 WaitForEncodedFrame(timestamp);
5421 timestamp += 9000;
5422 AdvanceTime(TimeDelta::Millis(100));
5423
5424 // Quality adaptation down.
5425 video_stream_encoder_->TriggerQualityLow();
5426
5427 // Adaptation has an effect.
5428 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5429 5000);
5430
5431 // Frame isn't dropped as initial frame dropper is disabled.
5432 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5433 WaitForEncodedFrame(timestamp);
5434 timestamp += 9000;
5435 AdvanceTime(TimeDelta::Millis(100));
5436
5437 // Quality adaptation up.
5438 video_stream_encoder_->TriggerQualityHigh();
5439
5440 // Adaptation has an effect.
5441 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5442 5000);
5443
5444 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5445 // Frame should not be dropped, as initial framedropper is off.
5446 WaitForEncodedFrame(timestamp);
5447
5448 video_stream_encoder_->Stop();
5449}
5450
Åsa Perssone644a032019-11-08 15:56:00 +01005451TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5452 webrtc::test::ScopedFieldTrials field_trials(
5453 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5454
5455 // Reset encoder for field trials to take effect.
5456 VideoEncoderConfig config = video_encoder_config_.Copy();
5457 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005458 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005459 ConfigureEncoder(std::move(config));
5460 fake_encoder_.SetQp(kQpLow);
5461
5462 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005463 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005464 source.set_adaptation_enabled(true);
5465 video_stream_encoder_->SetSource(&source,
5466 DegradationPreference::MAINTAIN_FRAMERATE);
5467
5468 // Start at low bitrate.
5469 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005470 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5471 DataRate::BitsPerSec(kLowBitrateBps),
5472 DataRate::BitsPerSec(kLowBitrateBps),
5473 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005474
5475 // Expect first frame to be dropped and resolution to be limited.
5476 const int kWidth = 1280;
5477 const int kHeight = 720;
5478 const int64_t kFrameIntervalMs = 100;
5479 int64_t timestamp_ms = kFrameIntervalMs;
5480 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5481 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005482 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5483 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005484
5485 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005486 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5487 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005488
5489 // Insert frames and advance |min_duration_ms|.
5490 for (size_t i = 1; i <= 10; i++) {
5491 timestamp_ms += kFrameIntervalMs;
5492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5493 WaitForEncodedFrame(timestamp_ms);
5494 }
5495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5496 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5497
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005498 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005499
5500 // Insert frame should trigger high BW and release quality limitation.
5501 timestamp_ms += kFrameIntervalMs;
5502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5503 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005504 // The ramp-up code involves the adaptation queue, give it time to execute.
5505 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005506 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005507 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005508
5509 // Frame should not be adapted.
5510 timestamp_ms += kFrameIntervalMs;
5511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5512 WaitForEncodedFrame(kWidth, kHeight);
5513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5514
5515 video_stream_encoder_->Stop();
5516}
5517
mflodmancc3d4422017-08-03 08:27:51 -07005518TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005519 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy066b5b62021-01-28 14:58:24 +01005520 webrtc::test::ScopedFieldTrials field_trials(
5521 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005522 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005523 source.set_adaptation_enabled(true);
5524 video_stream_encoder_->SetSource(&source,
5525 DegradationPreference::MAINTAIN_FRAMERATE);
5526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5527 DataRate::BitsPerSec(kTargetBitrateBps),
5528 DataRate::BitsPerSec(kTargetBitrateBps),
5529 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5530 fake_encoder_.SetQp(kQpHigh + 1);
5531 const int kWidth = 1280;
5532 const int kHeight = 720;
5533 const int64_t kFrameIntervalMs = 100;
5534 int64_t timestamp_ms = kFrameIntervalMs;
5535 for (size_t i = 1; i <= 100; i++) {
5536 timestamp_ms += kFrameIntervalMs;
5537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5538 WaitForEncodedFrame(timestamp_ms);
5539 }
5540 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5541 // for the first time.
5542 // TODO(eshr): We should avoid these waits by using threads with simulated
5543 // time.
5544 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5545 2000 * 2.5 * 2);
5546 timestamp_ms += kFrameIntervalMs;
5547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5548 WaitForEncodedFrame(timestamp_ms);
5549 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5550 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5552
5553 // Disable Quality scaling by turning off scaler on the encoder and
5554 // reconfiguring.
5555 fake_encoder_.SetQualityScaling(false);
5556 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5557 kMaxPayloadLength);
5558 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005559 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005560 // Since we turned off the quality scaler, the adaptations made by it are
5561 // removed.
5562 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5564
5565 video_stream_encoder_->Stop();
5566}
5567
5568TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005569 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5570 const int kTooSmallWidth = 10;
5571 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005573 DataRate::BitsPerSec(kTargetBitrateBps),
5574 DataRate::BitsPerSec(kTargetBitrateBps),
5575 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005576
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005577 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005578 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005579 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005580 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005581 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005582 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5583
5584 // Trigger adapt down, too small frame, expect no change.
5585 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005586 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005587 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005588 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5590 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5591
mflodmancc3d4422017-08-03 08:27:51 -07005592 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005593}
5594
mflodmancc3d4422017-08-03 08:27:51 -07005595TEST_F(VideoStreamEncoderTest,
5596 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005597 const int kTooSmallWidth = 10;
5598 const int kTooSmallHeight = 10;
5599 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005600 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005601 DataRate::BitsPerSec(kTargetBitrateBps),
5602 DataRate::BitsPerSec(kTargetBitrateBps),
5603 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005604
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005605 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005606 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005607 video_stream_encoder_->SetSource(&source,
5608 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005609 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5612
5613 // Trigger adapt down, expect limited framerate.
5614 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005615 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005616 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005617 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005618 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5621
5622 // Trigger adapt down, too small frame, expect no change.
5623 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005624 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07005625 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005626 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005627 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5629 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5630
mflodmancc3d4422017-08-03 08:27:51 -07005631 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005632}
5633
mflodmancc3d4422017-08-03 08:27:51 -07005634TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005635 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005637 DataRate::BitsPerSec(kTargetBitrateBps),
5638 DataRate::BitsPerSec(kTargetBitrateBps),
5639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005640 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005641 const int kFrameWidth = 1280;
5642 const int kFrameHeight = 720;
5643 video_source_.IncomingCapturedFrame(
5644 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005645 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005646 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005647}
5648
sprangb1ca0732017-02-01 08:38:12 -08005649// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005650TEST_F(VideoStreamEncoderTest,
5651 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005653 DataRate::BitsPerSec(kTargetBitrateBps),
5654 DataRate::BitsPerSec(kTargetBitrateBps),
5655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005656
5657 const int kFrameWidth = 1280;
5658 const int kFrameHeight = 720;
5659 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005660 // requested by
5661 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005662 video_source_.set_adaptation_enabled(true);
5663
5664 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005665 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005666 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005667
5668 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005669 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005670 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005671 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005672 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005673
asaperssonfab67072017-04-04 05:51:49 -07005674 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005675 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005676 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005677 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005678 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005679
mflodmancc3d4422017-08-03 08:27:51 -07005680 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005681}
sprangfe627f32017-03-29 08:24:59 -07005682
mflodmancc3d4422017-08-03 08:27:51 -07005683TEST_F(VideoStreamEncoderTest,
5684 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005685 const int kFrameWidth = 1280;
5686 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005687
Henrik Boström381d1092020-05-12 18:49:07 +02005688 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005689 DataRate::BitsPerSec(kTargetBitrateBps),
5690 DataRate::BitsPerSec(kTargetBitrateBps),
5691 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005693 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005694 video_source_.set_adaptation_enabled(true);
5695
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005696 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005697
5698 video_source_.IncomingCapturedFrame(
5699 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005700 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005701
5702 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005703 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005704
5705 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005706 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005707 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005708 video_source_.IncomingCapturedFrame(
5709 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005710 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005711 }
5712
5713 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005714 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005715 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005716 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005717 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005718 video_source_.IncomingCapturedFrame(
5719 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005720 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005721 ++num_frames_dropped;
5722 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005723 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005724 }
5725 }
5726
sprang4847ae62017-06-27 07:06:52 -07005727 // Add some slack to account for frames dropped by the frame dropper.
5728 const int kErrorMargin = 1;
5729 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005730 kErrorMargin);
5731
5732 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005733 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005734 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005735 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005736 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005737 video_source_.IncomingCapturedFrame(
5738 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005739 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005740 ++num_frames_dropped;
5741 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005742 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005743 }
5744 }
sprang4847ae62017-06-27 07:06:52 -07005745 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005746 kErrorMargin);
5747
5748 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005749 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005750 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005751 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005752 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005753 video_source_.IncomingCapturedFrame(
5754 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005755 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005756 ++num_frames_dropped;
5757 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005758 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005759 }
5760 }
sprang4847ae62017-06-27 07:06:52 -07005761 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005762 kErrorMargin);
5763
5764 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005765 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005766 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005767 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005768 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005769 video_source_.IncomingCapturedFrame(
5770 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005771 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005772 ++num_frames_dropped;
5773 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005774 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005775 }
5776 }
5777 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5778
mflodmancc3d4422017-08-03 08:27:51 -07005779 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005780}
5781
mflodmancc3d4422017-08-03 08:27:51 -07005782TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005783 const int kFramerateFps = 5;
5784 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005785 const int kFrameWidth = 1280;
5786 const int kFrameHeight = 720;
5787
sprang4847ae62017-06-27 07:06:52 -07005788 // Reconfigure encoder with two temporal layers and screensharing, which will
5789 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005790 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005791
Henrik Boström381d1092020-05-12 18:49:07 +02005792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005793 DataRate::BitsPerSec(kTargetBitrateBps),
5794 DataRate::BitsPerSec(kTargetBitrateBps),
5795 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005796 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005797 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005798 video_source_.set_adaptation_enabled(true);
5799
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005800 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005801
5802 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005803 rtc::VideoSinkWants last_wants;
5804 do {
5805 last_wants = video_source_.sink_wants();
5806
sprangc5d62e22017-04-02 23:53:04 -07005807 // Insert frames to get a new fps estimate...
5808 for (int j = 0; j < kFramerateFps; ++j) {
5809 video_source_.IncomingCapturedFrame(
5810 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005811 if (video_source_.last_sent_width()) {
5812 sink_.WaitForEncodedFrame(timestamp_ms);
5813 }
sprangc5d62e22017-04-02 23:53:04 -07005814 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005815 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07005816 }
5817 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07005818 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08005819 } while (video_source_.sink_wants().max_framerate_fps <
5820 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07005821
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005822 EXPECT_THAT(video_source_.sink_wants(),
5823 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07005824
mflodmancc3d4422017-08-03 08:27:51 -07005825 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005826}
asaperssonf7e294d2017-06-13 23:25:22 -07005827
mflodmancc3d4422017-08-03 08:27:51 -07005828TEST_F(VideoStreamEncoderTest,
5829 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005830 const int kWidth = 1280;
5831 const int kHeight = 720;
5832 const int64_t kFrameIntervalMs = 150;
5833 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02005834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005835 DataRate::BitsPerSec(kTargetBitrateBps),
5836 DataRate::BitsPerSec(kTargetBitrateBps),
5837 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005838
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005839 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005840 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07005841 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005842 video_stream_encoder_->SetSource(&source,
5843 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07005844 timestamp_ms += kFrameIntervalMs;
5845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005846 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005847 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07005848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5850 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5851
5852 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005853 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005854 timestamp_ms += kFrameIntervalMs;
5855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005856 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005857 EXPECT_THAT(source.sink_wants(),
5858 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07005859 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5860 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5861 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5862
5863 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005864 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005865 timestamp_ms += kFrameIntervalMs;
5866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005867 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005868 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5870 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5871 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5872
5873 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005874 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005875 timestamp_ms += kFrameIntervalMs;
5876 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005877 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005878 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5880 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5881 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5882
5883 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005884 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005885 timestamp_ms += kFrameIntervalMs;
5886 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005887 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005888 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005889 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5890 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5891 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5892
5893 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005894 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005895 timestamp_ms += kFrameIntervalMs;
5896 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005897 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005898 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005899 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5900 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5901 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5902
5903 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005904 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005905 timestamp_ms += kFrameIntervalMs;
5906 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005907 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005908 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005909 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5910 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5911 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5912
5913 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07005914 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005915 timestamp_ms += kFrameIntervalMs;
5916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005917 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005918 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005919 rtc::VideoSinkWants last_wants = source.sink_wants();
5920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5921 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5922 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5923
5924 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07005925 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07005926 timestamp_ms += kFrameIntervalMs;
5927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005928 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005929 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07005930 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5931 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5932 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5933
Evan Shrubsole64469032020-06-11 10:45:29 +02005934 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005935 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005936 timestamp_ms += kFrameIntervalMs;
5937 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005938 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005939 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005940 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5941 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5942 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5943
5944 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07005945 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005946 timestamp_ms += kFrameIntervalMs;
5947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005948 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005949 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005950 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5951 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5952 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5953
5954 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005955 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005956 timestamp_ms += kFrameIntervalMs;
5957 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005958 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005959 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005960 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5961 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5962 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5963
5964 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07005965 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005966 timestamp_ms += kFrameIntervalMs;
5967 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005968 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005969 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005970 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5971 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5972 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5973
5974 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005975 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005976 timestamp_ms += kFrameIntervalMs;
5977 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005978 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005979 EXPECT_THAT(source.sink_wants(), FpsMax());
5980 EXPECT_EQ(source.sink_wants().max_pixel_count,
5981 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07005982 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5983 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5984 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5985
5986 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005987 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005988 timestamp_ms += kFrameIntervalMs;
5989 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005990 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02005991 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07005992 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5993 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5994 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5995
Åsa Persson30ab0152019-08-27 12:22:33 +02005996 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07005997 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07005998 timestamp_ms += kFrameIntervalMs;
5999 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006000 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006001 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006002 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006003 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6005 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6006
6007 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006008 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006009 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006010 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6011
mflodmancc3d4422017-08-03 08:27:51 -07006012 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006013}
6014
mflodmancc3d4422017-08-03 08:27:51 -07006015TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006016 const int kWidth = 1280;
6017 const int kHeight = 720;
6018 const int64_t kFrameIntervalMs = 150;
6019 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006021 DataRate::BitsPerSec(kTargetBitrateBps),
6022 DataRate::BitsPerSec(kTargetBitrateBps),
6023 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006024
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006025 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006026 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006027 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006028 video_stream_encoder_->SetSource(&source,
6029 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006030 timestamp_ms += kFrameIntervalMs;
6031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006032 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006033 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6036 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6037 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6038 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6039 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6040
6041 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006042 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006043 timestamp_ms += kFrameIntervalMs;
6044 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006045 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006046 EXPECT_THAT(source.sink_wants(),
6047 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006048 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6049 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6050 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6051 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6052 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6053 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6054
6055 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006056 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006057 timestamp_ms += kFrameIntervalMs;
6058 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006059 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006060 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006061 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6062 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6063 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6064 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6065 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6066 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6067
6068 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006069 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006070 timestamp_ms += kFrameIntervalMs;
6071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006072 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006073 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006074 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006075 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6076 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6077 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6078 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6079 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6080
Evan Shrubsole64469032020-06-11 10:45:29 +02006081 // Trigger cpu adapt up, expect no change since QP is most limited.
6082 {
6083 // Store current sink wants since we expect no change and if there is no
6084 // change then last_wants() is not updated.
6085 auto previous_sink_wants = source.sink_wants();
6086 video_stream_encoder_->TriggerCpuUnderuse();
6087 timestamp_ms += kFrameIntervalMs;
6088 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6089 WaitForEncodedFrame(timestamp_ms);
6090 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6091 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6092 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6093 }
6094
6095 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6096 video_stream_encoder_->TriggerQualityHigh();
6097 timestamp_ms += kFrameIntervalMs;
6098 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6099 WaitForEncodedFrame(timestamp_ms);
6100 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6101 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6102 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6103 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6104 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6105 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6106 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6107
6108 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6109 // expect increased resolution (960x540@30fps).
6110 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006111 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006112 timestamp_ms += kFrameIntervalMs;
6113 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006114 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006115 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006116 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6118 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6119 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6120 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006121 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006122
Evan Shrubsole64469032020-06-11 10:45:29 +02006123 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6124 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006125 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006126 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006127 timestamp_ms += kFrameIntervalMs;
6128 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006129 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006130 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006131 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006132 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6133 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6134 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6135 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6136 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006137 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006138
6139 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006140 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006141 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006142 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006143 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006144
mflodmancc3d4422017-08-03 08:27:51 -07006145 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006146}
6147
mflodmancc3d4422017-08-03 08:27:51 -07006148TEST_F(VideoStreamEncoderTest,
6149 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006150 const int kWidth = 640;
6151 const int kHeight = 360;
6152 const int kFpsLimit = 15;
6153 const int64_t kFrameIntervalMs = 150;
6154 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006155 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006156 DataRate::BitsPerSec(kTargetBitrateBps),
6157 DataRate::BitsPerSec(kTargetBitrateBps),
6158 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006159
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006160 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006161 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006162 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006163 video_stream_encoder_->SetSource(&source,
6164 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006165 timestamp_ms += kFrameIntervalMs;
6166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006167 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006168 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006169 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6170 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6171 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6172 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6173 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6174 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6175
6176 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006177 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006178 timestamp_ms += kFrameIntervalMs;
6179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006180 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006181 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006182 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6183 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6184 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6185 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6186 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6187 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6188
6189 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006190 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006193 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006194 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006195 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006196 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006197 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6198 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6199 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6200 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6201
Evan Shrubsole64469032020-06-11 10:45:29 +02006202 // Trigger cpu adapt up, expect no change because quality is most limited.
6203 {
6204 auto previous_sink_wants = source.sink_wants();
6205 // Store current sink wants since we expect no change ind if there is no
6206 // change then last__wants() is not updated.
6207 video_stream_encoder_->TriggerCpuUnderuse();
6208 timestamp_ms += kFrameIntervalMs;
6209 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6210 WaitForEncodedFrame(timestamp_ms);
6211 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6212 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6213 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6214 }
6215
6216 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6217 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006218 timestamp_ms += kFrameIntervalMs;
6219 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006220 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006221 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006222 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6223 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6224 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006225 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6226 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6227 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006228
Evan Shrubsole64469032020-06-11 10:45:29 +02006229 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006230 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006231 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006232 timestamp_ms += kFrameIntervalMs;
6233 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006234 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006235 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6237 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6238 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6239 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6240 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006241 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006242
6243 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006244 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006245 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006246 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006247 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006248
mflodmancc3d4422017-08-03 08:27:51 -07006249 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006250}
6251
mflodmancc3d4422017-08-03 08:27:51 -07006252TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006253 const int kFrameWidth = 1920;
6254 const int kFrameHeight = 1080;
6255 // 3/4 of 1920.
6256 const int kAdaptedFrameWidth = 1440;
6257 // 3/4 of 1080 rounded down to multiple of 4.
6258 const int kAdaptedFrameHeight = 808;
6259 const int kFramerate = 24;
6260
Henrik Boström381d1092020-05-12 18:49:07 +02006261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006262 DataRate::BitsPerSec(kTargetBitrateBps),
6263 DataRate::BitsPerSec(kTargetBitrateBps),
6264 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006265 // Trigger reconfigure encoder (without resetting the entire instance).
6266 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006267 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6268 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006269 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006270 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006271 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006272 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006273 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006274 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006275
6276 video_source_.set_adaptation_enabled(true);
6277
6278 video_source_.IncomingCapturedFrame(
6279 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006280 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006281
6282 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006283 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006284 video_source_.IncomingCapturedFrame(
6285 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006286 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006287
mflodmancc3d4422017-08-03 08:27:51 -07006288 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006289}
6290
mflodmancc3d4422017-08-03 08:27:51 -07006291TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006292 const int kFrameWidth = 1280;
6293 const int kFrameHeight = 720;
6294 const int kLowFps = 2;
6295 const int kHighFps = 30;
6296
Henrik Boström381d1092020-05-12 18:49:07 +02006297 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006298 DataRate::BitsPerSec(kTargetBitrateBps),
6299 DataRate::BitsPerSec(kTargetBitrateBps),
6300 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006301
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006302 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006303 max_framerate_ = kLowFps;
6304
6305 // Insert 2 seconds of 2fps video.
6306 for (int i = 0; i < kLowFps * 2; ++i) {
6307 video_source_.IncomingCapturedFrame(
6308 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6309 WaitForEncodedFrame(timestamp_ms);
6310 timestamp_ms += 1000 / kLowFps;
6311 }
6312
6313 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006314 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006315 DataRate::BitsPerSec(kTargetBitrateBps),
6316 DataRate::BitsPerSec(kTargetBitrateBps),
6317 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006318 video_source_.IncomingCapturedFrame(
6319 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6320 WaitForEncodedFrame(timestamp_ms);
6321 timestamp_ms += 1000 / kLowFps;
6322
6323 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6324
6325 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006326 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006327 const int kFrameIntervalMs = 1000 / kHighFps;
6328 max_framerate_ = kHighFps;
6329 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6330 video_source_.IncomingCapturedFrame(
6331 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6332 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6333 // be dropped if the encoder hans't been updated with the new higher target
6334 // framerate yet, causing it to overshoot the target bitrate and then
6335 // suffering the wrath of the media optimizer.
6336 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6337 timestamp_ms += kFrameIntervalMs;
6338 }
6339
6340 // Don expect correct measurement just yet, but it should be higher than
6341 // before.
6342 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6343
mflodmancc3d4422017-08-03 08:27:51 -07006344 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006345}
6346
mflodmancc3d4422017-08-03 08:27:51 -07006347TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006348 const int kFrameWidth = 1280;
6349 const int kFrameHeight = 720;
6350 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006351 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006352 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006353 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006354
Henrik Boström381d1092020-05-12 18:49:07 +02006355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006356 DataRate::BitsPerSec(kTargetBitrateBps),
6357 DataRate::BitsPerSec(kTargetBitrateBps),
6358 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006360
6361 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006362 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006363 video_source_.IncomingCapturedFrame(
6364 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6365 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006366 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006367
6368 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6370 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6371 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006372
6373 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006374 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006375 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006376
Per Kjellanderdcef6412020-10-07 15:09:05 +02006377 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006378 video_source_.IncomingCapturedFrame(
6379 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6380 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006381 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006382
mflodmancc3d4422017-08-03 08:27:51 -07006383 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006384}
ilnik6b826ef2017-06-16 06:53:48 -07006385
Niels Möller4db138e2018-04-19 09:04:13 +02006386TEST_F(VideoStreamEncoderTest,
6387 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6388 const int kFrameWidth = 1280;
6389 const int kFrameHeight = 720;
6390 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006391 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006392 DataRate::BitsPerSec(kTargetBitrateBps),
6393 DataRate::BitsPerSec(kTargetBitrateBps),
6394 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006395 video_source_.IncomingCapturedFrame(
6396 CreateFrame(1, kFrameWidth, kFrameHeight));
6397 WaitForEncodedFrame(1);
6398 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6399 .low_encode_usage_threshold_percent,
6400 default_options.low_encode_usage_threshold_percent);
6401 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6402 .high_encode_usage_threshold_percent,
6403 default_options.high_encode_usage_threshold_percent);
6404 video_stream_encoder_->Stop();
6405}
6406
6407TEST_F(VideoStreamEncoderTest,
6408 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6409 const int kFrameWidth = 1280;
6410 const int kFrameHeight = 720;
6411 CpuOveruseOptions hardware_options;
6412 hardware_options.low_encode_usage_threshold_percent = 150;
6413 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006414 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006415
Henrik Boström381d1092020-05-12 18:49:07 +02006416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006417 DataRate::BitsPerSec(kTargetBitrateBps),
6418 DataRate::BitsPerSec(kTargetBitrateBps),
6419 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006420 video_source_.IncomingCapturedFrame(
6421 CreateFrame(1, kFrameWidth, kFrameHeight));
6422 WaitForEncodedFrame(1);
6423 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6424 .low_encode_usage_threshold_percent,
6425 hardware_options.low_encode_usage_threshold_percent);
6426 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6427 .high_encode_usage_threshold_percent,
6428 hardware_options.high_encode_usage_threshold_percent);
6429 video_stream_encoder_->Stop();
6430}
6431
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006432TEST_F(VideoStreamEncoderTest,
6433 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6434 const int kFrameWidth = 1280;
6435 const int kFrameHeight = 720;
6436
6437 const CpuOveruseOptions default_options;
6438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6439 DataRate::BitsPerSec(kTargetBitrateBps),
6440 DataRate::BitsPerSec(kTargetBitrateBps),
6441 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6442 video_source_.IncomingCapturedFrame(
6443 CreateFrame(1, kFrameWidth, kFrameHeight));
6444 WaitForEncodedFrame(1);
6445 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6446 .low_encode_usage_threshold_percent,
6447 default_options.low_encode_usage_threshold_percent);
6448 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6449 .high_encode_usage_threshold_percent,
6450 default_options.high_encode_usage_threshold_percent);
6451
6452 CpuOveruseOptions hardware_options;
6453 hardware_options.low_encode_usage_threshold_percent = 150;
6454 hardware_options.high_encode_usage_threshold_percent = 200;
6455 fake_encoder_.SetIsHardwareAccelerated(true);
6456
6457 video_source_.IncomingCapturedFrame(
6458 CreateFrame(2, kFrameWidth, kFrameHeight));
6459 WaitForEncodedFrame(2);
6460
6461 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6462 .low_encode_usage_threshold_percent,
6463 hardware_options.low_encode_usage_threshold_percent);
6464 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6465 .high_encode_usage_threshold_percent,
6466 hardware_options.high_encode_usage_threshold_percent);
6467
6468 video_stream_encoder_->Stop();
6469}
6470
Niels Möller6bb5ab92019-01-11 11:11:10 +01006471TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6472 const int kFrameWidth = 320;
6473 const int kFrameHeight = 240;
6474 const int kFps = 30;
6475 const int kTargetBitrateBps = 120000;
6476 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6477
Henrik Boström381d1092020-05-12 18:49:07 +02006478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006479 DataRate::BitsPerSec(kTargetBitrateBps),
6480 DataRate::BitsPerSec(kTargetBitrateBps),
6481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006482
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006483 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006484 max_framerate_ = kFps;
6485
6486 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6487 fake_encoder_.SimulateOvershoot(1.0);
6488 int num_dropped = 0;
6489 for (int i = 0; i < kNumFramesInRun; ++i) {
6490 video_source_.IncomingCapturedFrame(
6491 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6492 // Wait up to two frame durations for a frame to arrive.
6493 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6494 ++num_dropped;
6495 }
6496 timestamp_ms += 1000 / kFps;
6497 }
6498
Erik Språnga8d48ab2019-02-08 14:17:40 +01006499 // Framerate should be measured to be near the expected target rate.
6500 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6501
6502 // Frame drops should be within 5% of expected 0%.
6503 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006504
6505 // Make encoder produce frames at double the expected bitrate during 3 seconds
6506 // of video, verify number of drops. Rate needs to be slightly changed in
6507 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006508 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006509 const RateControlSettings trials =
6510 RateControlSettings::ParseFromFieldTrials();
6511 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006512 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006513 // frame dropping since the adjuter will try to just lower the target
6514 // bitrate rather than drop frames. If network headroom can be used, it
6515 // doesn't push back as hard so we don't need quite as much overshoot.
6516 // These numbers are unfortunately a bit magical but there's not trivial
6517 // way to algebraically infer them.
6518 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6519 overshoot_factor = 2.4;
6520 } else {
6521 overshoot_factor = 4.0;
6522 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006523 }
6524 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006526 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6527 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6528 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006529 num_dropped = 0;
6530 for (int i = 0; i < kNumFramesInRun; ++i) {
6531 video_source_.IncomingCapturedFrame(
6532 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6533 // Wait up to two frame durations for a frame to arrive.
6534 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6535 ++num_dropped;
6536 }
6537 timestamp_ms += 1000 / kFps;
6538 }
6539
Henrik Boström381d1092020-05-12 18:49:07 +02006540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006541 DataRate::BitsPerSec(kTargetBitrateBps),
6542 DataRate::BitsPerSec(kTargetBitrateBps),
6543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006544
6545 // Target framerate should be still be near the expected target, despite
6546 // the frame drops.
6547 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6548
6549 // Frame drops should be within 5% of expected 50%.
6550 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006551
6552 video_stream_encoder_->Stop();
6553}
6554
6555TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6556 const int kFrameWidth = 320;
6557 const int kFrameHeight = 240;
6558 const int kActualInputFps = 24;
6559 const int kTargetBitrateBps = 120000;
6560
6561 ASSERT_GT(max_framerate_, kActualInputFps);
6562
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006563 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006564 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006565 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006566 DataRate::BitsPerSec(kTargetBitrateBps),
6567 DataRate::BitsPerSec(kTargetBitrateBps),
6568 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006569
6570 // Insert 3 seconds of video, with an input fps lower than configured max.
6571 for (int i = 0; i < kActualInputFps * 3; ++i) {
6572 video_source_.IncomingCapturedFrame(
6573 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6574 // Wait up to two frame durations for a frame to arrive.
6575 WaitForEncodedFrame(timestamp_ms);
6576 timestamp_ms += 1000 / kActualInputFps;
6577 }
6578
6579 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6580
6581 video_stream_encoder_->Stop();
6582}
6583
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006584TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006585 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006586 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006587 DataRate::BitsPerSec(kTargetBitrateBps),
6588 DataRate::BitsPerSec(kTargetBitrateBps),
6589 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006590
6591 fake_encoder_.BlockNextEncode();
6592 video_source_.IncomingCapturedFrame(
6593 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6594 WaitForEncodedFrame(1);
6595 // On the very first frame full update should be forced.
6596 rect = fake_encoder_.GetLastUpdateRect();
6597 EXPECT_EQ(rect.offset_x, 0);
6598 EXPECT_EQ(rect.offset_y, 0);
6599 EXPECT_EQ(rect.height, codec_height_);
6600 EXPECT_EQ(rect.width, codec_width_);
6601 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6602 // call to ContinueEncode.
6603 video_source_.IncomingCapturedFrame(
6604 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6605 ExpectDroppedFrame();
6606 video_source_.IncomingCapturedFrame(
6607 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6608 ExpectDroppedFrame();
6609 fake_encoder_.ContinueEncode();
6610 WaitForEncodedFrame(3);
6611 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6612 rect = fake_encoder_.GetLastUpdateRect();
6613 EXPECT_EQ(rect.offset_x, 1);
6614 EXPECT_EQ(rect.offset_y, 0);
6615 EXPECT_EQ(rect.width, 10);
6616 EXPECT_EQ(rect.height, 1);
6617
6618 video_source_.IncomingCapturedFrame(
6619 CreateFrameWithUpdatedPixel(4, nullptr, 0));
6620 WaitForEncodedFrame(4);
6621 // Previous frame was encoded, so no accumulation should happen.
6622 rect = fake_encoder_.GetLastUpdateRect();
6623 EXPECT_EQ(rect.offset_x, 0);
6624 EXPECT_EQ(rect.offset_y, 0);
6625 EXPECT_EQ(rect.width, 1);
6626 EXPECT_EQ(rect.height, 1);
6627
6628 video_stream_encoder_->Stop();
6629}
6630
Erik Språngd7329ca2019-02-21 21:19:53 +01006631TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02006632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006633 DataRate::BitsPerSec(kTargetBitrateBps),
6634 DataRate::BitsPerSec(kTargetBitrateBps),
6635 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006636
6637 // First frame is always keyframe.
6638 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6639 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01006640 EXPECT_THAT(
6641 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006642 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006643
6644 // Insert delta frame.
6645 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6646 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01006647 EXPECT_THAT(
6648 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006649 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006650
6651 // Request next frame be a key-frame.
6652 video_stream_encoder_->SendKeyFrame();
6653 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6654 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01006655 EXPECT_THAT(
6656 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006657 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006658
6659 video_stream_encoder_->Stop();
6660}
6661
6662TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
6663 // Setup simulcast with three streams.
6664 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006666 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6667 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6668 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006669 // Wait for all three layers before triggering event.
6670 sink_.SetNumExpectedLayers(3);
6671
6672 // First frame is always keyframe.
6673 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6674 WaitForEncodedFrame(1);
6675 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006676 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6677 VideoFrameType::kVideoFrameKey,
6678 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006679
6680 // Insert delta frame.
6681 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6682 WaitForEncodedFrame(2);
6683 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006684 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6685 VideoFrameType::kVideoFrameDelta,
6686 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006687
6688 // Request next frame be a key-frame.
6689 // Only first stream is configured to produce key-frame.
6690 video_stream_encoder_->SendKeyFrame();
6691 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6692 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006693
6694 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6695 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006696 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006697 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006698 VideoFrameType::kVideoFrameKey,
6699 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006700
6701 video_stream_encoder_->Stop();
6702}
6703
6704TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6705 // Configure internal source factory and setup test again.
6706 encoder_factory_.SetHasInternalSource(true);
6707 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006709 DataRate::BitsPerSec(kTargetBitrateBps),
6710 DataRate::BitsPerSec(kTargetBitrateBps),
6711 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006712
6713 // Call encoder directly, simulating internal source where encoded frame
6714 // callback in VideoStreamEncoder is called despite no OnFrame().
6715 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6716 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006717 EXPECT_THAT(
6718 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006719 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006720
Niels Möller8f7ce222019-03-21 15:43:58 +01006721 const std::vector<VideoFrameType> kDeltaFrame = {
6722 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006723 // Need to set timestamp manually since manually for injected frame.
6724 VideoFrame frame = CreateFrame(101, nullptr);
6725 frame.set_timestamp(101);
6726 fake_encoder_.InjectFrame(frame, false);
6727 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006728 EXPECT_THAT(
6729 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006730 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006731
6732 // Request key-frame. The forces a dummy frame down into the encoder.
6733 fake_encoder_.ExpectNullFrame();
6734 video_stream_encoder_->SendKeyFrame();
6735 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006736 EXPECT_THAT(
6737 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006738 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006739
6740 video_stream_encoder_->Stop();
6741}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006742
6743TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6744 // Configure internal source factory and setup test again.
6745 encoder_factory_.SetHasInternalSource(true);
6746 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006747 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006748 DataRate::BitsPerSec(kTargetBitrateBps),
6749 DataRate::BitsPerSec(kTargetBitrateBps),
6750 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006751
6752 int64_t timestamp = 1;
6753 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006754 image.SetEncodedData(
6755 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006756 image.capture_time_ms_ = ++timestamp;
6757 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6758 const int64_t kEncodeFinishDelayMs = 10;
6759 image.timing_.encode_start_ms = timestamp;
6760 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6761 fake_encoder_.InjectEncodedImage(image);
6762 // Wait for frame without incrementing clock.
6763 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6764 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6765 // capture timestamp should be kEncodeFinishDelayMs in the past.
6766 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006767 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006768
6769 video_stream_encoder_->Stop();
6770}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006771
6772TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006773 // SPS contains VUI with restrictions on the maximum number of reordered
6774 // pictures, there is no need to rewrite the bitstream to enable faster
6775 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006776 ResetEncoder("H264", 1, 1, 1, false);
6777
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6779 DataRate::BitsPerSec(kTargetBitrateBps),
6780 DataRate::BitsPerSec(kTargetBitrateBps),
6781 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6782 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006783
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006784 fake_encoder_.SetEncodedImageData(
6785 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006786
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006787 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6788 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006789
6790 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6791 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006792
6793 video_stream_encoder_->Stop();
6794}
6795
6796TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006797 // SPS does not contain VUI, the bitstream is will be rewritten with added
6798 // VUI with restrictions on the maximum number of reordered pictures to
6799 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006800 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6801 0x00, 0x00, 0x03, 0x03, 0xF4,
6802 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006803 ResetEncoder("H264", 1, 1, 1, false);
6804
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006805 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6806 DataRate::BitsPerSec(kTargetBitrateBps),
6807 DataRate::BitsPerSec(kTargetBitrateBps),
6808 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6809 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006810
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006811 fake_encoder_.SetEncodedImageData(
6812 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006813
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006814 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6815 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006816
6817 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6818 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006819
6820 video_stream_encoder_->Stop();
6821}
6822
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006823TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
6824 const int kFrameWidth = 1280;
6825 const int kFrameHeight = 720;
6826 const int kTargetBitrateBps = 300000; // To low for HD resolution.
6827
Henrik Boström381d1092020-05-12 18:49:07 +02006828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006829 DataRate::BitsPerSec(kTargetBitrateBps),
6830 DataRate::BitsPerSec(kTargetBitrateBps),
6831 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006832 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6833
6834 // Insert a first video frame. It should be dropped because of downscale in
6835 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006836 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006837 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6838 frame.set_rotation(kVideoRotation_270);
6839 video_source_.IncomingCapturedFrame(frame);
6840
6841 ExpectDroppedFrame();
6842
6843 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006844 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006845 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6846 frame.set_rotation(kVideoRotation_90);
6847 video_source_.IncomingCapturedFrame(frame);
6848
6849 WaitForEncodedFrame(timestamp_ms);
6850 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
6851
6852 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006853 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02006854 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
6855 frame.set_rotation(kVideoRotation_180);
6856 video_source_.IncomingCapturedFrame(frame);
6857
6858 WaitForEncodedFrame(timestamp_ms);
6859 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
6860
6861 video_stream_encoder_->Stop();
6862}
6863
Erik Språng5056af02019-09-02 15:53:11 +02006864TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
6865 const int kFrameWidth = 320;
6866 const int kFrameHeight = 180;
6867
6868 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02006869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006870 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
6871 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
6872 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02006873 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006874 /*rtt_ms=*/0,
6875 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006876
6877 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006878 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02006879 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
6880 frame.set_rotation(kVideoRotation_270);
6881 video_source_.IncomingCapturedFrame(frame);
6882 WaitForEncodedFrame(timestamp_ms);
6883
6884 // Set a target rate below the minimum allowed by the codec settings.
6885 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006886 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
6887 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02006888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02006889 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02006890 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02006891 /*link_allocation=*/target_rate,
6892 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006893 /*rtt_ms=*/0,
6894 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02006895 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6896
6897 // Target bitrate and bandwidth allocation should both be capped at min_rate.
6898 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6899 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006900 DataRate allocation_sum =
6901 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02006902 EXPECT_EQ(min_rate, allocation_sum);
6903 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
6904
6905 video_stream_encoder_->Stop();
6906}
6907
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01006908TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02006909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006910 DataRate::BitsPerSec(kTargetBitrateBps),
6911 DataRate::BitsPerSec(kTargetBitrateBps),
6912 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006913 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006914 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006915 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6916 WaitForEncodedFrame(1);
6917
6918 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
6919 ASSERT_TRUE(prev_rate_settings.has_value());
6920 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
6921 kDefaultFramerate);
6922
6923 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
6924 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
6925 timestamp_ms += 1000 / kDefaultFramerate;
6926 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6927 WaitForEncodedFrame(timestamp_ms);
6928 }
6929 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
6930 kDefaultFramerate);
6931 // Capture larger frame to trigger a reconfigure.
6932 codec_height_ *= 2;
6933 codec_width_ *= 2;
6934 timestamp_ms += 1000 / kDefaultFramerate;
6935 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
6936 WaitForEncodedFrame(timestamp_ms);
6937
6938 EXPECT_EQ(2, sink_.number_of_reconfigurations());
6939 auto current_rate_settings =
6940 fake_encoder_.GetAndResetLastRateControlSettings();
6941 // Ensure we have actually reconfigured twice
6942 // The rate settings should have been set again even though
6943 // they haven't changed.
6944 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02006945 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02006946
6947 video_stream_encoder_->Stop();
6948}
6949
philipeld9cc8c02019-09-16 14:53:40 +02006950struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02006951 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
6952 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
6953 MOCK_METHOD(void,
6954 RequestEncoderSwitch,
6955 (const webrtc::SdpVideoFormat& format),
6956 (override));
philipeld9cc8c02019-09-16 14:53:40 +02006957};
6958
6959TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
6960 constexpr int kDontCare = 100;
6961
6962 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
6963 video_send_config_.encoder_settings.encoder_switch_request_callback =
6964 &switch_callback;
6965 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
6966 encoder_config.codec_type = kVideoCodecVP8;
6967 webrtc::test::ScopedFieldTrials field_trial(
6968 "WebRTC-NetworkCondition-EncoderSwitch/"
6969 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
6970 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
6971
6972 // Reset encoder for new configuration to take effect.
6973 ConfigureEncoder(std::move(encoder_config));
6974
6975 // Send one frame to trigger ReconfigureEncoder.
6976 video_source_.IncomingCapturedFrame(
6977 CreateFrame(kDontCare, kDontCare, kDontCare));
6978
6979 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01006980 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
6981 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02006982 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01006983 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02006984
Henrik Boström381d1092020-05-12 18:49:07 +02006985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006986 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
6987 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
6988 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02006989 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01006990 /*rtt_ms=*/0,
6991 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02006992 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02006993
6994 video_stream_encoder_->Stop();
6995}
6996
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01006997TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
6998 constexpr int kDontCare = 100;
6999
7000 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7001 video_send_config_.encoder_settings.encoder_switch_request_callback =
7002 &switch_callback;
7003 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7004 encoder_config.codec_type = kVideoCodecVP8;
7005 webrtc::test::ScopedFieldTrials field_trial(
7006 "WebRTC-NetworkCondition-EncoderSwitch/"
7007 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
7008 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7009
7010 // Reset encoder for new configuration to take effect.
7011 ConfigureEncoder(std::move(encoder_config));
7012
7013 // Send one frame to trigger ReconfigureEncoder.
7014 video_source_.IncomingCapturedFrame(
7015 CreateFrame(kDontCare, kDontCare, kDontCare));
7016
7017 using Config = EncoderSwitchRequestCallback::Config;
7018 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
7019 .Times(0);
7020
Henrik Boström381d1092020-05-12 18:49:07 +02007021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01007022 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
7023 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
7024 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
7025 /*fraction_lost=*/0,
7026 /*rtt_ms=*/0,
7027 /*cwnd_reduce_ratio=*/0);
7028
7029 video_stream_encoder_->Stop();
7030}
7031
philipeld9cc8c02019-09-16 14:53:40 +02007032TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
7033 constexpr int kSufficientBitrateToNotDrop = 1000;
7034 constexpr int kHighRes = 500;
7035 constexpr int kLowRes = 100;
7036
7037 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7038 video_send_config_.encoder_settings.encoder_switch_request_callback =
7039 &switch_callback;
7040 webrtc::test::ScopedFieldTrials field_trial(
7041 "WebRTC-NetworkCondition-EncoderSwitch/"
7042 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
7043 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7044 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7045 encoder_config.codec_type = kVideoCodecH264;
7046
7047 // Reset encoder for new configuration to take effect.
7048 ConfigureEncoder(std::move(encoder_config));
7049
7050 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7051 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7052 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007054 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7055 /*stable_target_bitrate=*/
7056 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7057 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02007058 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007059 /*rtt_ms=*/0,
7060 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02007061
7062 // Send one frame to trigger ReconfigureEncoder.
7063 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
7064 WaitForEncodedFrame(1);
7065
7066 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01007067 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
7068 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02007069 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01007070 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02007071
7072 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
7073 WaitForEncodedFrame(2);
7074
7075 video_stream_encoder_->Stop();
7076}
7077
philipel9b058032020-02-10 11:30:00 +01007078TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7079 constexpr int kDontCare = 100;
7080 StrictMock<MockEncoderSelector> encoder_selector;
7081 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7082 &fake_encoder_, &encoder_selector);
7083 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7084
7085 // Reset encoder for new configuration to take effect.
7086 ConfigureEncoder(video_encoder_config_.Copy());
7087
7088 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7089
7090 video_source_.IncomingCapturedFrame(
7091 CreateFrame(kDontCare, kDontCare, kDontCare));
7092 video_stream_encoder_->Stop();
7093
7094 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7095 // to it's factory, so in order for the encoder instance in the
7096 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7097 // reset the |video_stream_encoder_| here.
7098 video_stream_encoder_.reset();
7099}
7100
7101TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7102 constexpr int kDontCare = 100;
7103
7104 NiceMock<MockEncoderSelector> encoder_selector;
7105 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7106 video_send_config_.encoder_settings.encoder_switch_request_callback =
7107 &switch_callback;
7108 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7109 &fake_encoder_, &encoder_selector);
7110 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7111
7112 // Reset encoder for new configuration to take effect.
7113 ConfigureEncoder(video_encoder_config_.Copy());
7114
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007115 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007116 .WillByDefault(Return(SdpVideoFormat("AV1")));
7117 EXPECT_CALL(switch_callback,
7118 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7119 Field(&SdpVideoFormat::name, "AV1"))));
7120
Henrik Boström381d1092020-05-12 18:49:07 +02007121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007122 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7123 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7124 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007125 /*fraction_lost=*/0,
7126 /*rtt_ms=*/0,
7127 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007128 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007129
7130 video_stream_encoder_->Stop();
7131}
7132
7133TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7134 constexpr int kSufficientBitrateToNotDrop = 1000;
7135 constexpr int kDontCare = 100;
7136
7137 NiceMock<MockVideoEncoder> video_encoder;
7138 NiceMock<MockEncoderSelector> encoder_selector;
7139 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7140 video_send_config_.encoder_settings.encoder_switch_request_callback =
7141 &switch_callback;
7142 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7143 &video_encoder, &encoder_selector);
7144 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7145
7146 // Reset encoder for new configuration to take effect.
7147 ConfigureEncoder(video_encoder_config_.Copy());
7148
7149 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7150 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7151 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007153 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7154 /*stable_target_bitrate=*/
7155 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7156 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007157 /*fraction_lost=*/0,
7158 /*rtt_ms=*/0,
7159 /*cwnd_reduce_ratio=*/0);
7160
7161 ON_CALL(video_encoder, Encode(_, _))
7162 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7163 ON_CALL(encoder_selector, OnEncoderBroken())
7164 .WillByDefault(Return(SdpVideoFormat("AV2")));
7165
7166 rtc::Event encode_attempted;
7167 EXPECT_CALL(switch_callback,
7168 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7169 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7170 EXPECT_EQ(format.name, "AV2");
7171 encode_attempted.Set();
7172 });
7173
7174 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7175 encode_attempted.Wait(3000);
7176
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007177 AdvanceTime(TimeDelta::Millis(0));
7178
philipel9b058032020-02-10 11:30:00 +01007179 video_stream_encoder_->Stop();
7180
7181 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7182 // to it's factory, so in order for the encoder instance in the
7183 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7184 // reset the |video_stream_encoder_| here.
7185 video_stream_encoder_.reset();
7186}
7187
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007188TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007189 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007190 const int kFrameWidth = 320;
7191 const int kFrameHeight = 180;
7192
7193 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007194 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007196 /*target_bitrate=*/rate,
7197 /*stable_target_bitrate=*/rate,
7198 /*link_allocation=*/rate,
7199 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007200 /*rtt_ms=*/0,
7201 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007202
7203 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007204 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007205 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7206 frame.set_rotation(kVideoRotation_270);
7207 video_source_.IncomingCapturedFrame(frame);
7208 WaitForEncodedFrame(timestamp_ms);
7209 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7210
7211 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007212 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007213 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007214 /*target_bitrate=*/new_stable_rate,
7215 /*stable_target_bitrate=*/new_stable_rate,
7216 /*link_allocation=*/rate,
7217 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007218 /*rtt_ms=*/0,
7219 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7221 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7222 video_stream_encoder_->Stop();
7223}
7224
7225TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007226 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007227 const int kFrameWidth = 320;
7228 const int kFrameHeight = 180;
7229
7230 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007231 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007233 /*target_bitrate=*/rate,
7234 /*stable_target_bitrate=*/rate,
7235 /*link_allocation=*/rate,
7236 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007237 /*rtt_ms=*/0,
7238 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007239
7240 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007241 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007242 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7243 frame.set_rotation(kVideoRotation_270);
7244 video_source_.IncomingCapturedFrame(frame);
7245 WaitForEncodedFrame(timestamp_ms);
7246 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7247
7248 // Set a higher target rate without changing the link_allocation. Should not
7249 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007250 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007252 /*target_bitrate=*/rate,
7253 /*stable_target_bitrate=*/new_stable_rate,
7254 /*link_allocation=*/rate,
7255 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007256 /*rtt_ms=*/0,
7257 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7259 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7260 video_stream_encoder_->Stop();
7261}
7262
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007263TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7264 test::ScopedFieldTrials field_trials(
7265 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7266 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7267 const int kFramerateFps = 30;
7268 const int kWidth = 1920;
7269 const int kHeight = 1080;
7270 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7271 // Works on screenshare mode.
7272 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7273 // We rely on the automatic resolution adaptation, but we handle framerate
7274 // adaptation manually by mocking the stats proxy.
7275 video_source_.set_adaptation_enabled(true);
7276
7277 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007279 DataRate::BitsPerSec(kTargetBitrateBps),
7280 DataRate::BitsPerSec(kTargetBitrateBps),
7281 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007282 video_stream_encoder_->SetSource(&video_source_,
7283 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007284 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007285
7286 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7287 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7288
7289 // Pass enough frames with the full update to trigger animation detection.
7290 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007291 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007292 frame.set_ntp_time_ms(timestamp_ms);
7293 frame.set_timestamp_us(timestamp_ms * 1000);
7294 video_source_.IncomingCapturedFrame(frame);
7295 WaitForEncodedFrame(timestamp_ms);
7296 }
7297
7298 // Resolution should be limited.
7299 rtc::VideoSinkWants expected;
7300 expected.max_framerate_fps = kFramerateFps;
7301 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007302 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007303
7304 // Pass one frame with no known update.
7305 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007306 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007307 frame.set_ntp_time_ms(timestamp_ms);
7308 frame.set_timestamp_us(timestamp_ms * 1000);
7309 frame.clear_update_rect();
7310
7311 video_source_.IncomingCapturedFrame(frame);
7312 WaitForEncodedFrame(timestamp_ms);
7313
7314 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007315 EXPECT_THAT(video_source_.sink_wants(),
7316 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007317
7318 video_stream_encoder_->Stop();
7319}
7320
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007321TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7322 const int kWidth = 720; // 540p adapted down.
7323 const int kHeight = 405;
7324 const int kNumFrames = 3;
7325 // Works on screenshare mode.
7326 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7327 /*num_spatial_layers=*/2, /*screenshare=*/true);
7328
7329 video_source_.set_adaptation_enabled(true);
7330
7331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7332 DataRate::BitsPerSec(kTargetBitrateBps),
7333 DataRate::BitsPerSec(kTargetBitrateBps),
7334 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7335
7336 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7337
7338 // Pass enough frames with the full update to trigger animation detection.
7339 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007340 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007341 frame.set_ntp_time_ms(timestamp_ms);
7342 frame.set_timestamp_us(timestamp_ms * 1000);
7343 video_source_.IncomingCapturedFrame(frame);
7344 WaitForEncodedFrame(timestamp_ms);
7345 }
7346
7347 video_stream_encoder_->Stop();
7348}
7349
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007350TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7351 const float downscale_factors[] = {4.0, 2.0, 1.0};
7352 const int number_layers =
7353 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7354 VideoEncoderConfig config;
7355 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7356 for (int i = 0; i < number_layers; ++i) {
7357 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7358 config.simulcast_layers[i].active = true;
7359 }
7360 config.video_stream_factory =
7361 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7362 "VP8", /*max qp*/ 56, /*screencast*/ false,
7363 /*screenshare enabled*/ false);
7364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7365 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7366 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7367 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7368
7369 // First initialization.
7370 // Encoder should be initialized. Next frame should be key frame.
7371 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7372 sink_.SetNumExpectedLayers(number_layers);
7373 int64_t timestamp_ms = kFrameIntervalMs;
7374 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7375 WaitForEncodedFrame(timestamp_ms);
7376 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7377 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7378 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7379 VideoFrameType::kVideoFrameKey,
7380 VideoFrameType::kVideoFrameKey}));
7381
7382 // Disable top layer.
7383 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7384 config.simulcast_layers[number_layers - 1].active = false;
7385 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7386 sink_.SetNumExpectedLayers(number_layers - 1);
7387 timestamp_ms += kFrameIntervalMs;
7388 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7389 WaitForEncodedFrame(timestamp_ms);
7390 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7391 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7392 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7393 VideoFrameType::kVideoFrameDelta,
7394 VideoFrameType::kVideoFrameDelta}));
7395
7396 // Re-enable top layer.
7397 // Encoder should be re-initialized. Next frame should be key frame.
7398 config.simulcast_layers[number_layers - 1].active = true;
7399 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7400 sink_.SetNumExpectedLayers(number_layers);
7401 timestamp_ms += kFrameIntervalMs;
7402 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7403 WaitForEncodedFrame(timestamp_ms);
7404 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7405 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7406 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7407 VideoFrameType::kVideoFrameKey,
7408 VideoFrameType::kVideoFrameKey}));
7409
7410 // Top layer max rate change.
7411 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7412 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7413 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7414 sink_.SetNumExpectedLayers(number_layers);
7415 timestamp_ms += kFrameIntervalMs;
7416 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7417 WaitForEncodedFrame(timestamp_ms);
7418 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7419 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7420 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7421 VideoFrameType::kVideoFrameDelta,
7422 VideoFrameType::kVideoFrameDelta}));
7423
7424 // Top layer resolution change.
7425 // Encoder should be re-initialized. Next frame should be key frame.
7426 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7427 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7428 sink_.SetNumExpectedLayers(number_layers);
7429 timestamp_ms += kFrameIntervalMs;
7430 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7431 WaitForEncodedFrame(timestamp_ms);
7432 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7433 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7434 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7435 VideoFrameType::kVideoFrameKey,
7436 VideoFrameType::kVideoFrameKey}));
7437 video_stream_encoder_->Stop();
7438}
7439
perkj26091b12016-09-01 01:17:40 -07007440} // namespace webrtc