blob: d74ebe84adb676d9ea1a0ce9c46e51622091c5ce [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Per512ecb32016-09-23 15:52:06 +020016#include <utility>
17
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020018#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020019#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020020#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010021#include "api/test/mock_video_encoder.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080022#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020024#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020025#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010026#include "api/video/video_bitrate_allocation.h"
Elad Alon370f93a2019-06-11 14:57:57 +020027#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020028#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010029#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020030#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010031#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020032#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070033#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080034#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020035#include "media/engine/webrtc_video_engine.h"
Sergey Silkin86684962018-03-28 19:32:37 +020036#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020037#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010038#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020039#include "rtc_base/event.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020040#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020041#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020043#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010044#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020045#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020046#include "test/encoder_settings.h"
47#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020048#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010049#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "test/gmock.h"
51#include "test/gtest.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020053#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020054#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070055
56namespace webrtc {
57
sprang57c2fff2017-01-16 06:24:02 -080058using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020059using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020060using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020061using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020062using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020063using ::testing::Ge;
64using ::testing::Gt;
65using ::testing::Le;
66using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010067using ::testing::Matcher;
68using ::testing::NiceMock;
69using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010070using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020071using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080072
perkj803d97f2016-11-01 11:45:46 -070073namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020074const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010075const int kQpLow = 1;
76const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020077const int kMinFramerateFps = 2;
78const int kMinBalancedFramerateFps = 7;
79const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080080const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010081const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020082const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010083const uint32_t kSimulcastTargetBitrateBps = 3150000;
84const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080085const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070086const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020088const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020089const VideoEncoder::ResolutionBitrateLimits
90 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
91const VideoEncoder::ResolutionBitrateLimits
92 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -080093
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020094uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
95 0x00, 0x00, 0x03, 0x03, 0xF4,
96 0x05, 0x03, 0xC7, 0xE0, 0x1B,
97 0x41, 0x10, 0x8D, 0x00};
98
perkj803d97f2016-11-01 11:45:46 -070099class TestBuffer : public webrtc::I420Buffer {
100 public:
101 TestBuffer(rtc::Event* event, int width, int height)
102 : I420Buffer(width, height), event_(event) {}
103
104 private:
105 friend class rtc::RefCountedObject<TestBuffer>;
106 ~TestBuffer() override {
107 if (event_)
108 event_->Set();
109 }
110 rtc::Event* const event_;
111};
112
Noah Richards51db4212019-06-12 06:59:12 -0700113// A fake native buffer that can't be converted to I420.
114class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
115 public:
116 FakeNativeBuffer(rtc::Event* event, int width, int height)
117 : event_(event), width_(width), height_(height) {}
118 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
119 int width() const override { return width_; }
120 int height() const override { return height_; }
121 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
122 return nullptr;
123 }
124
125 private:
126 friend class rtc::RefCountedObject<FakeNativeBuffer>;
127 ~FakeNativeBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132 const int width_;
133 const int height_;
134};
135
Evan Shrubsole895556e2020-10-05 09:15:13 +0200136// A fake native buffer that is backed by an NV12 buffer.
137class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
140 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
141
142 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
143 int width() const override { return nv12_buffer_->width(); }
144 int height() const override { return nv12_buffer_->height(); }
145 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
146 return nv12_buffer_->ToI420();
147 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200148 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
149 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
150 if (absl::c_find(types, Type::kNV12) != types.end()) {
151 return nv12_buffer_;
152 }
153 return nullptr;
154 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200155 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
156
157 private:
158 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
159 ~FakeNV12NativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
164 rtc::Event* const event_;
165};
166
Niels Möller7dc26b72017-12-06 10:27:48 +0100167class CpuOveruseDetectorProxy : public OveruseFrameDetector {
168 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200169 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
170 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200171 last_target_framerate_fps_(-1),
172 framerate_updated_event_(true /* manual_reset */,
173 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100174 virtual ~CpuOveruseDetectorProxy() {}
175
176 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200177 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100178 last_target_framerate_fps_ = framerate_fps;
179 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200180 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100181 }
182
183 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200184 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100185 return last_target_framerate_fps_;
186 }
187
Niels Möller4db138e2018-04-19 09:04:13 +0200188 CpuOveruseOptions GetOptions() { return options_; }
189
Henrik Boström381d1092020-05-12 18:49:07 +0200190 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
191
Niels Möller7dc26b72017-12-06 10:27:48 +0100192 private:
Markus Handella3765182020-07-08 13:13:32 +0200193 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100194 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200195 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100196};
197
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200198class FakeVideoSourceRestrictionsListener
199 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200200 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200201 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200202 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200203 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200204 RTC_DCHECK(was_restrictions_updated_);
205 }
206
207 rtc::Event* restrictions_updated_event() {
208 return &restrictions_updated_event_;
209 }
210
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200211 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200212 void OnVideoSourceRestrictionsUpdated(
213 VideoSourceRestrictions restrictions,
214 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200215 rtc::scoped_refptr<Resource> reason,
216 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200217 was_restrictions_updated_ = true;
218 restrictions_updated_event_.Set();
219 }
220
221 private:
222 bool was_restrictions_updated_;
223 rtc::Event restrictions_updated_event_;
224};
225
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200226auto WantsFps(Matcher<int> fps_matcher) {
227 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
228 fps_matcher);
229}
230
231auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
232 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
233 AllOf(max_pixel_matcher, Gt(0)));
234}
235
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200236auto ResolutionMax() {
237 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200238 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200239 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
240 Eq(absl::nullopt)));
241}
242
243auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200244 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200245}
246
247auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200248 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200249}
250
251auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200253}
254
255auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200256 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200257}
258
259auto FpsMaxResolutionMax() {
260 return AllOf(FpsMax(), ResolutionMax());
261}
262
263auto UnlimitedSinkWants() {
264 return AllOf(FpsUnlimited(), ResolutionMax());
265}
266
267auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
268 Matcher<int> fps_range_matcher;
269
270 if (last_frame_pixels <= 320 * 240) {
271 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200272 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200273 fps_range_matcher = AllOf(Ge(10), Le(15));
274 } else if (last_frame_pixels <= 640 * 480) {
275 fps_range_matcher = Ge(15);
276 } else {
277 fps_range_matcher = Eq(kDefaultFramerate);
278 }
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_range_matcher);
281}
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
284 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
285 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
286}
287
288auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
289 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
290}
291
292auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
293 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
294}
295
296auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
297 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
298 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
299}
300
301auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
302 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
303 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
304}
305
306auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
307 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
308 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
309}
310
311auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
312 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
313 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
314}
315
mflodmancc3d4422017-08-03 08:27:51 -0700316class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700317 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200318 VideoStreamEncoderUnderTest(TimeController* time_controller,
319 TaskQueueFactory* task_queue_factory,
320 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100321 const VideoStreamEncoderSettings& settings,
322 VideoStreamEncoder::BitrateAllocationCallbackType
323 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200324 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100325 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200326 stats_proxy,
327 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200328 std::unique_ptr<OveruseFrameDetector>(
329 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100330 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100331 task_queue_factory,
332 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200333 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200334 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200335 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200336 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200337 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200338 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200339 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200340 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100341 }
perkj803d97f2016-11-01 11:45:46 -0700342
Henrik Boström381d1092020-05-12 18:49:07 +0200343 void SetSourceAndWaitForRestrictionsUpdated(
344 rtc::VideoSourceInterface<VideoFrame>* source,
345 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200346 FakeVideoSourceRestrictionsListener listener;
347 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200348 SetSource(source, degradation_preference);
349 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200350 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200351 }
352
353 void SetSourceAndWaitForFramerateUpdated(
354 rtc::VideoSourceInterface<VideoFrame>* source,
355 const DegradationPreference& degradation_preference) {
356 overuse_detector_proxy_->framerate_updated_event()->Reset();
357 SetSource(source, degradation_preference);
358 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
359 }
360
361 void OnBitrateUpdatedAndWaitForManagedResources(
362 DataRate target_bitrate,
363 DataRate stable_target_bitrate,
364 DataRate link_allocation,
365 uint8_t fraction_lost,
366 int64_t round_trip_time_ms,
367 double cwnd_reduce_ratio) {
368 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
369 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
370 // Bitrate is updated on the encoder queue.
371 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200372 }
373
kthelgason2fc52542017-03-03 00:24:41 -0800374 // This is used as a synchronisation mechanism, to make sure that the
375 // encoder queue is not blocked before we start sending it frames.
376 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100377 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200378 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800379 ASSERT_TRUE(event.Wait(5000));
380 }
381
Henrik Boström91aa7322020-04-28 12:24:33 +0200382 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200383 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200384 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200385 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200386 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200387 event.Set();
388 });
389 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200390 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200391 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200392
Henrik Boström91aa7322020-04-28 12:24:33 +0200393 void TriggerCpuUnderuse() {
394 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200395 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200396 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200397 event.Set();
398 });
399 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200400 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200401 }
kthelgason876222f2016-11-29 01:44:11 -0800402
Henrik Boström91aa7322020-04-28 12:24:33 +0200403 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200404 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200405 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200406 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200407 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 event.Set();
409 });
410 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200411 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200412 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200413 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200415 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200416 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 event.Set();
418 });
419 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 }
422
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200423 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100424 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200425 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
426 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200427 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700428};
429
Noah Richards51db4212019-06-12 06:59:12 -0700430// Simulates simulcast behavior and makes highest stream resolutions divisible
431// by 4.
432class CroppingVideoStreamFactory
433 : public VideoEncoderConfig::VideoStreamFactoryInterface {
434 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200435 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700436
437 private:
438 std::vector<VideoStream> CreateEncoderStreams(
439 int width,
440 int height,
441 const VideoEncoderConfig& encoder_config) override {
442 std::vector<VideoStream> streams = test::CreateVideoStreams(
443 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700444 return streams;
445 }
Noah Richards51db4212019-06-12 06:59:12 -0700446};
447
sprangb1ca0732017-02-01 08:38:12 -0800448class AdaptingFrameForwarder : public test::FrameForwarder {
449 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 explicit AdaptingFrameForwarder(TimeController* time_controller)
451 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700452 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800453
454 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200455 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800456 adaptation_enabled_ = enabled;
457 }
458
asaperssonfab67072017-04-04 05:51:49 -0700459 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200460 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800461 return adaptation_enabled_;
462 }
463
Henrik Boström1124ed12021-02-25 10:30:39 +0100464 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
465 // the resolution or frame rate was different than it is currently. If
466 // something else is modified, such as encoder resolutions, but the resolution
467 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700468 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200469 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700470 return last_wants_;
471 }
472
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200473 absl::optional<int> last_sent_width() const { return last_width_; }
474 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800475
sprangb1ca0732017-02-01 08:38:12 -0800476 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
478 time_controller_->AdvanceTime(TimeDelta::Millis(0));
479
sprangb1ca0732017-02-01 08:38:12 -0800480 int cropped_width = 0;
481 int cropped_height = 0;
482 int out_width = 0;
483 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700484 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200485 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
486 << "w=" << video_frame.width()
487 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700488 if (adapter_.AdaptFrameResolution(
489 video_frame.width(), video_frame.height(),
490 video_frame.timestamp_us() * 1000, &cropped_width,
491 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100492 VideoFrame adapted_frame =
493 VideoFrame::Builder()
494 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
495 nullptr, out_width, out_height))
496 .set_timestamp_rtp(99)
497 .set_timestamp_ms(99)
498 .set_rotation(kVideoRotation_0)
499 .build();
sprangc5d62e22017-04-02 23:53:04 -0700500 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100501 if (video_frame.has_update_rect()) {
502 adapted_frame.set_update_rect(
503 video_frame.update_rect().ScaleWithFrame(
504 video_frame.width(), video_frame.height(), 0, 0,
505 video_frame.width(), video_frame.height(), out_width,
506 out_height));
507 }
sprangc5d62e22017-04-02 23:53:04 -0700508 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800509 last_width_.emplace(adapted_frame.width());
510 last_height_.emplace(adapted_frame.height());
511 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200512 last_width_ = absl::nullopt;
513 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700514 }
sprangb1ca0732017-02-01 08:38:12 -0800515 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200516 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800517 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800518 last_width_.emplace(video_frame.width());
519 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800520 }
521 }
522
523 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
524 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200525 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100526 rtc::VideoSinkWants prev_wants = sink_wants_locked();
527 bool did_adapt =
528 prev_wants.max_pixel_count != wants.max_pixel_count ||
529 prev_wants.target_pixel_count != wants.target_pixel_count ||
530 prev_wants.max_framerate_fps != wants.max_framerate_fps;
531 if (did_adapt) {
532 last_wants_ = prev_wants;
533 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100534 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200535 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800536 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200537
538 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800539 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200540 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
541 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200542 absl::optional<int> last_width_;
543 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800544};
sprangc5d62e22017-04-02 23:53:04 -0700545
Niels Möller213618e2018-07-24 09:29:58 +0200546// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700547class MockableSendStatisticsProxy : public SendStatisticsProxy {
548 public:
549 MockableSendStatisticsProxy(Clock* clock,
550 const VideoSendStream::Config& config,
551 VideoEncoderConfig::ContentType content_type)
552 : SendStatisticsProxy(clock, config, content_type) {}
553
554 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200555 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700556 if (mock_stats_)
557 return *mock_stats_;
558 return SendStatisticsProxy::GetStats();
559 }
560
Niels Möller213618e2018-07-24 09:29:58 +0200561 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200562 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200563 if (mock_stats_)
564 return mock_stats_->input_frame_rate;
565 return SendStatisticsProxy::GetInputFrameRate();
566 }
sprangc5d62e22017-04-02 23:53:04 -0700567 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200568 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700569 mock_stats_.emplace(stats);
570 }
571
572 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200573 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700574 mock_stats_.reset();
575 }
576
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200577 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
578 on_frame_dropped_ = std::move(callback);
579 }
580
sprangc5d62e22017-04-02 23:53:04 -0700581 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200582 void OnFrameDropped(DropReason reason) override {
583 SendStatisticsProxy::OnFrameDropped(reason);
584 if (on_frame_dropped_)
585 on_frame_dropped_(reason);
586 }
587
Markus Handella3765182020-07-08 13:13:32 +0200588 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200589 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200590 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700591};
592
philipel9b058032020-02-10 11:30:00 +0100593class MockEncoderSelector
594 : public VideoEncoderFactory::EncoderSelectorInterface {
595 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200596 MOCK_METHOD(void,
597 OnCurrentEncoder,
598 (const SdpVideoFormat& format),
599 (override));
600 MOCK_METHOD(absl::optional<SdpVideoFormat>,
601 OnAvailableBitrate,
602 (const DataRate& rate),
603 (override));
604 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100605};
606
perkj803d97f2016-11-01 11:45:46 -0700607} // namespace
608
mflodmancc3d4422017-08-03 08:27:51 -0700609class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700610 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200611 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700612
mflodmancc3d4422017-08-03 08:27:51 -0700613 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700614 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700615 codec_width_(320),
616 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200617 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200618 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200619 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700620 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200621 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700622 video_send_config_,
623 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200624 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700625
626 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700627 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700628 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200629 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800630 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200631 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200632 video_send_config_.rtp.payload_name = "FAKE";
633 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700634
Per512ecb32016-09-23 15:52:06 +0200635 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200636 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200637 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
638 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
639 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100640 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700641
Niels Möllerf1338562018-04-26 09:51:47 +0200642 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800643 }
644
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100645 void ConfigureEncoder(
646 VideoEncoderConfig video_encoder_config,
647 VideoStreamEncoder::BitrateAllocationCallbackType
648 allocation_callback_type =
649 VideoStreamEncoder::BitrateAllocationCallbackType::
650 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700651 if (video_stream_encoder_)
652 video_stream_encoder_->Stop();
653 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200654 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100655 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700656 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
657 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700658 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700659 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
660 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200661 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700662 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800663 }
664
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100665 void ResetEncoder(const std::string& payload_name,
666 size_t num_streams,
667 size_t num_temporal_layers,
668 unsigned char num_spatial_layers,
669 bool screenshare,
670 VideoStreamEncoder::BitrateAllocationCallbackType
671 allocation_callback_type =
672 VideoStreamEncoder::BitrateAllocationCallbackType::
673 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200674 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800675
676 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200677 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
678 num_streams, &video_encoder_config);
679 for (auto& layer : video_encoder_config.simulcast_layers) {
680 layer.num_temporal_layers = num_temporal_layers;
681 layer.max_framerate = kDefaultFramerate;
682 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100683 video_encoder_config.max_bitrate_bps =
684 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700685 video_encoder_config.content_type =
686 screenshare ? VideoEncoderConfig::ContentType::kScreen
687 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700688 if (payload_name == "VP9") {
689 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
690 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200691 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700692 video_encoder_config.encoder_specific_settings =
693 new rtc::RefCountedObject<
694 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
695 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100696 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700697 }
698
sprang57c2fff2017-01-16 06:24:02 -0800699 VideoFrame CreateFrame(int64_t ntp_time_ms,
700 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100701 VideoFrame frame =
702 VideoFrame::Builder()
703 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
704 destruction_event, codec_width_, codec_height_))
705 .set_timestamp_rtp(99)
706 .set_timestamp_ms(99)
707 .set_rotation(kVideoRotation_0)
708 .build();
sprang57c2fff2017-01-16 06:24:02 -0800709 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700710 return frame;
711 }
712
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100713 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
714 rtc::Event* destruction_event,
715 int offset_x) const {
716 VideoFrame frame =
717 VideoFrame::Builder()
718 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
719 destruction_event, codec_width_, codec_height_))
720 .set_timestamp_rtp(99)
721 .set_timestamp_ms(99)
722 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100723 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100724 .build();
725 frame.set_ntp_time_ms(ntp_time_ms);
726 return frame;
727 }
728
sprang57c2fff2017-01-16 06:24:02 -0800729 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100730 VideoFrame frame =
731 VideoFrame::Builder()
732 .set_video_frame_buffer(
733 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
734 .set_timestamp_rtp(99)
735 .set_timestamp_ms(99)
736 .set_rotation(kVideoRotation_0)
737 .build();
sprang57c2fff2017-01-16 06:24:02 -0800738 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700739 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700740 return frame;
741 }
742
Evan Shrubsole895556e2020-10-05 09:15:13 +0200743 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
744 VideoFrame frame =
745 VideoFrame::Builder()
746 .set_video_frame_buffer(NV12Buffer::Create(width, height))
747 .set_timestamp_rtp(99)
748 .set_timestamp_ms(99)
749 .set_rotation(kVideoRotation_0)
750 .build();
751 frame.set_ntp_time_ms(ntp_time_ms);
752 frame.set_timestamp_us(ntp_time_ms * 1000);
753 return frame;
754 }
755
Noah Richards51db4212019-06-12 06:59:12 -0700756 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
757 rtc::Event* destruction_event,
758 int width,
759 int height) const {
760 VideoFrame frame =
761 VideoFrame::Builder()
762 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
763 destruction_event, width, height))
764 .set_timestamp_rtp(99)
765 .set_timestamp_ms(99)
766 .set_rotation(kVideoRotation_0)
767 .build();
768 frame.set_ntp_time_ms(ntp_time_ms);
769 return frame;
770 }
771
Evan Shrubsole895556e2020-10-05 09:15:13 +0200772 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
773 rtc::Event* destruction_event,
774 int width,
775 int height) const {
776 VideoFrame frame = VideoFrame::Builder()
777 .set_video_frame_buffer(
778 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
779 destruction_event, width, height))
780 .set_timestamp_rtp(99)
781 .set_timestamp_ms(99)
782 .set_rotation(kVideoRotation_0)
783 .build();
784 frame.set_ntp_time_ms(ntp_time_ms);
785 return frame;
786 }
787
Noah Richards51db4212019-06-12 06:59:12 -0700788 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
789 rtc::Event* destruction_event) const {
790 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
791 codec_height_);
792 }
793
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100794 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100796 DataRate::BitsPerSec(kTargetBitrateBps),
797 DataRate::BitsPerSec(kTargetBitrateBps),
798 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100799
800 video_source_.IncomingCapturedFrame(
801 CreateFrame(1, codec_width_, codec_height_));
802 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200803 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100804 }
805
sprang4847ae62017-06-27 07:06:52 -0700806 void WaitForEncodedFrame(int64_t expected_ntp_time) {
807 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200808 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700809 }
810
811 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
812 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 return ok;
815 }
816
817 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
818 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200819 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700820 }
821
822 void ExpectDroppedFrame() {
823 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700825 }
826
827 bool WaitForFrame(int64_t timeout_ms) {
828 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200829 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700830 return ok;
831 }
832
perkj26091b12016-09-01 01:17:40 -0700833 class TestEncoder : public test::FakeEncoder {
834 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 explicit TestEncoder(TimeController* time_controller)
836 : FakeEncoder(time_controller->GetClock()),
837 time_controller_(time_controller) {
838 RTC_DCHECK(time_controller_);
839 }
perkj26091b12016-09-01 01:17:40 -0700840
asaperssonfab67072017-04-04 05:51:49 -0700841 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200842 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700843 return config_;
844 }
845
846 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200847 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700848 block_next_encode_ = true;
849 }
850
Erik Språngaed30702018-11-05 12:57:17 +0100851 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200852 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200853 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100854 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100855 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100856 info.scaling_settings = VideoEncoder::ScalingSettings(
857 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100858 }
859 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100860 for (int i = 0; i < kMaxSpatialLayers; ++i) {
861 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100862 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100863 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100864 for (int tid = 0; tid < num_layers; ++tid)
865 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100866 }
867 }
Erik Språngaed30702018-11-05 12:57:17 +0100868 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200869
870 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100871 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200872 info.apply_alignment_to_all_simulcast_layers =
873 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200874 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100875 return info;
kthelgason876222f2016-11-29 01:44:11 -0800876 }
877
Erik Språngb7cb7b52019-02-26 15:52:33 +0100878 int32_t RegisterEncodeCompleteCallback(
879 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200880 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100881 encoded_image_callback_ = callback;
882 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
883 }
884
perkjfa10b552016-10-02 23:45:26 -0700885 void ContinueEncode() { continue_encode_event_.Set(); }
886
887 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
888 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200889 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700890 EXPECT_EQ(timestamp_, timestamp);
891 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
892 }
893
kthelgason2fc52542017-03-03 00:24:41 -0800894 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200895 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800896 quality_scaling_ = b;
897 }
kthelgasonad9010c2017-02-14 00:46:51 -0800898
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100899 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200900 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100901 requested_resolution_alignment_ = requested_resolution_alignment;
902 }
903
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200904 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
905 MutexLock lock(&local_mutex_);
906 apply_alignment_to_all_simulcast_layers_ = b;
907 }
908
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100909 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200910 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100911 is_hardware_accelerated_ = is_hardware_accelerated;
912 }
913
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100914 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
915 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100917 temporal_layers_supported_[spatial_idx] = supported;
918 }
919
Sergey Silkin6456e352019-07-08 17:56:40 +0200920 void SetResolutionBitrateLimits(
921 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200922 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200923 resolution_bitrate_limits_ = thresholds;
924 }
925
sprangfe627f32017-03-29 08:24:59 -0700926 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200927 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700928 force_init_encode_failed_ = force_failure;
929 }
930
Niels Möller6bb5ab92019-01-11 11:11:10 +0100931 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200932 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100933 rate_factor_ = rate_factor;
934 }
935
Erik Språngd7329ca2019-02-21 21:19:53 +0100936 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200937 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100938 return last_framerate_;
939 }
940
Erik Språngd7329ca2019-02-21 21:19:53 +0100941 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200942 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100943 return last_update_rect_;
944 }
945
Niels Möller87e2d782019-03-07 10:18:23 +0100946 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200947 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100948 return last_frame_types_;
949 }
950
951 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100952 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100953 keyframe ? VideoFrameType::kVideoFrameKey
954 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 {
Markus Handella3765182020-07-08 13:13:32 +0200956 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100957 last_frame_types_ = frame_type;
958 }
Niels Möllerb859b322019-03-07 12:40:01 +0100959 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100960 }
961
Erik Språngb7cb7b52019-02-26 15:52:33 +0100962 void InjectEncodedImage(const EncodedImage& image) {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +0200964 encoded_image_callback_->OnEncodedImage(image, nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100965 }
966
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200967 void SetEncodedImageData(
968 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200969 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200970 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200971 }
972
Erik Språngd7329ca2019-02-21 21:19:53 +0100973 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200974 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100975 expect_null_frame_ = true;
976 }
977
Erik Språng5056af02019-09-02 15:53:11 +0200978 absl::optional<VideoEncoder::RateControlParameters>
979 GetAndResetLastRateControlSettings() {
980 auto settings = last_rate_control_settings_;
981 last_rate_control_settings_.reset();
982 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100983 }
984
Evan Shrubsole895556e2020-10-05 09:15:13 +0200985 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
986 MutexLock lock(&local_mutex_);
987 return last_input_pixel_format_;
988 }
989
Sergey Silkin5ee69672019-07-02 14:18:34 +0200990 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +0200991 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +0200992 return num_encoder_initializations_;
993 }
994
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200995 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +0200996 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +0200997 return num_set_rates_;
998 }
999
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001000 VideoCodec video_codec() const {
1001 MutexLock lock(&local_mutex_);
1002 return video_codec_;
1003 }
1004
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001005 void SetPreferredPixelFormats(
1006 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1007 pixel_formats) {
1008 MutexLock lock(&local_mutex_);
1009 preferred_pixel_formats_ = std::move(pixel_formats);
1010 }
1011
perkjfa10b552016-10-02 23:45:26 -07001012 private:
perkj26091b12016-09-01 01:17:40 -07001013 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001014 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001015 bool block_encode;
1016 {
Markus Handella3765182020-07-08 13:13:32 +02001017 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001018 if (expect_null_frame_) {
1019 EXPECT_EQ(input_image.timestamp(), 0u);
1020 EXPECT_EQ(input_image.width(), 1);
1021 last_frame_types_ = *frame_types;
1022 expect_null_frame_ = false;
1023 } else {
1024 EXPECT_GT(input_image.timestamp(), timestamp_);
1025 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1026 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1027 }
perkj26091b12016-09-01 01:17:40 -07001028
1029 timestamp_ = input_image.timestamp();
1030 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001031 last_input_width_ = input_image.width();
1032 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001033 block_encode = block_next_encode_;
1034 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001035 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001036 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001037 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001038 }
Niels Möllerb859b322019-03-07 12:40:01 +01001039 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001040 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001041 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001042
perkj26091b12016-09-01 01:17:40 -07001043 return result;
1044 }
1045
Niels Möller08ae7ce2020-09-23 15:58:12 +02001046 CodecSpecificInfo EncodeHook(
1047 EncodedImage& encoded_image,
1048 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001049 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001050 {
1051 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001052 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001053 }
1054 MutexLock lock(&local_mutex_);
1055 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001056 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001057 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001058 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001059 }
1060
sprangfe627f32017-03-29 08:24:59 -07001061 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001062 const Settings& settings) override {
1063 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001064
Markus Handella3765182020-07-08 13:13:32 +02001065 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001066 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001067
1068 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001069 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001070
Erik Språng82fad3d2018-03-21 09:57:23 +01001071 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001072 // Simulate setting up temporal layers, in order to validate the life
1073 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001074 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001075 frame_buffer_controller_ =
1076 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001077 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 if (force_init_encode_failed_) {
1079 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001080 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001081 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001082
Erik Språngb7cb7b52019-02-26 15:52:33 +01001083 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001084 return res;
1085 }
1086
Erik Språngb7cb7b52019-02-26 15:52:33 +01001087 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001088 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001089 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1090 initialized_ = EncoderState::kUninitialized;
1091 return FakeEncoder::Release();
1092 }
1093
Erik Språng16cb8f52019-04-12 13:59:09 +02001094 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001096 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001097 VideoBitrateAllocation adjusted_rate_allocation;
1098 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1099 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001100 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001101 adjusted_rate_allocation.SetBitrate(
1102 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001103 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001104 rate_factor_));
1105 }
1106 }
1107 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001108 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001109 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001110 RateControlParameters adjusted_paramters = parameters;
1111 adjusted_paramters.bitrate = adjusted_rate_allocation;
1112 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001113 }
1114
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001115 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001116 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001117 enum class EncoderState {
1118 kUninitialized,
1119 kInitializationFailed,
1120 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001121 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1122 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001123 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001124 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1125 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1126 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1127 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1128 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1129 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001130 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1131 false;
Markus Handella3765182020-07-08 13:13:32 +02001132 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001133 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1134 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001135 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001136 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001137 absl::optional<bool>
1138 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001139 local_mutex_);
1140 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1141 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1142 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001143 absl::optional<VideoEncoder::RateControlParameters>
1144 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001145 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1146 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001147 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001148 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001149 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1150 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001151 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001152 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001153 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001154 RTC_GUARDED_BY(local_mutex_);
1155 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001156 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001157 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1158 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001159 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1160 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001161 };
1162
mflodmancc3d4422017-08-03 08:27:51 -07001163 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001164 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001165 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1166 : time_controller_(time_controller), test_encoder_(test_encoder) {
1167 RTC_DCHECK(time_controller_);
1168 }
perkj26091b12016-09-01 01:17:40 -07001169
perkj26091b12016-09-01 01:17:40 -07001170 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001171 EXPECT_TRUE(
1172 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1173 }
1174
1175 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1176 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001177 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001178 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001179 return false;
perkj26091b12016-09-01 01:17:40 -07001180 {
Markus Handella3765182020-07-08 13:13:32 +02001181 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001182 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001183 }
1184 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001185 return true;
perkj26091b12016-09-01 01:17:40 -07001186 }
1187
sprangb1ca0732017-02-01 08:38:12 -08001188 void WaitForEncodedFrame(uint32_t expected_width,
1189 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001190 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001191 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001192 }
1193
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001194 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001195 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001196 uint32_t width = 0;
1197 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001198 {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001200 width = last_width_;
1201 height = last_height_;
1202 }
1203 EXPECT_EQ(expected_height, height);
1204 EXPECT_EQ(expected_width, width);
1205 }
1206
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001207 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1208 VideoRotation rotation;
1209 {
Markus Handella3765182020-07-08 13:13:32 +02001210 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001211 rotation = last_rotation_;
1212 }
1213 EXPECT_EQ(expected_rotation, rotation);
1214 }
1215
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001216 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001217
sprangc5d62e22017-04-02 23:53:04 -07001218 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001219 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1220 bool ret = encoded_frame_event_.Wait(timeout_ms);
1221 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1222 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001223 }
1224
perkj26091b12016-09-01 01:17:40 -07001225 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001226 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001227 expect_frames_ = false;
1228 }
1229
asaperssonfab67072017-04-04 05:51:49 -07001230 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001231 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001232 return number_of_reconfigurations_;
1233 }
1234
asaperssonfab67072017-04-04 05:51:49 -07001235 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001236 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001237 return min_transmit_bitrate_bps_;
1238 }
1239
Erik Språngd7329ca2019-02-21 21:19:53 +01001240 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001241 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001242 num_expected_layers_ = num_layers;
1243 }
1244
Erik Språngb7cb7b52019-02-26 15:52:33 +01001245 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001246 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001247 return last_capture_time_ms_;
1248 }
1249
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001250 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001251 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001252 return std::move(last_encoded_image_data_);
1253 }
1254
Per Kjellanderdcef6412020-10-07 15:09:05 +02001255 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1256 MutexLock lock(&mutex_);
1257 return last_bitrate_allocation_;
1258 }
1259
1260 int number_of_bitrate_allocations() const {
1261 MutexLock lock(&mutex_);
1262 return number_of_bitrate_allocations_;
1263 }
1264
Per Kjellandera9434842020-10-15 17:53:22 +02001265 VideoLayersAllocation GetLastVideoLayersAllocation() {
1266 MutexLock lock(&mutex_);
1267 return last_layers_allocation_;
1268 }
1269
1270 int number_of_layers_allocations() const {
1271 MutexLock lock(&mutex_);
1272 return number_of_layers_allocations_;
1273 }
1274
perkj26091b12016-09-01 01:17:40 -07001275 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001276 Result OnEncodedImage(
1277 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001278 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001279 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001280 EXPECT_TRUE(expect_frames_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001281 last_encoded_image_data_ = std::vector<uint8_t>(
1282 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001283 uint32_t timestamp = encoded_image.Timestamp();
1284 if (last_timestamp_ != timestamp) {
1285 num_received_layers_ = 1;
1286 } else {
1287 ++num_received_layers_;
1288 }
1289 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001290 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001291 last_width_ = encoded_image._encodedWidth;
1292 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001293 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001294 if (num_received_layers_ == num_expected_layers_) {
1295 encoded_frame_event_.Set();
1296 }
sprangb1ca0732017-02-01 08:38:12 -08001297 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001298 }
1299
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001300 void OnEncoderConfigurationChanged(
1301 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001302 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001303 VideoEncoderConfig::ContentType content_type,
1304 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001305 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001306 ++number_of_reconfigurations_;
1307 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1308 }
1309
Per Kjellanderdcef6412020-10-07 15:09:05 +02001310 void OnBitrateAllocationUpdated(
1311 const VideoBitrateAllocation& allocation) override {
1312 MutexLock lock(&mutex_);
1313 ++number_of_bitrate_allocations_;
1314 last_bitrate_allocation_ = allocation;
1315 }
1316
Per Kjellandera9434842020-10-15 17:53:22 +02001317 void OnVideoLayersAllocationUpdated(
1318 VideoLayersAllocation allocation) override {
1319 MutexLock lock(&mutex_);
1320 ++number_of_layers_allocations_;
1321 last_layers_allocation_ = allocation;
1322 rtc::StringBuilder log;
1323 for (const auto& layer : allocation.active_spatial_layers) {
1324 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1325 << "[";
1326 for (const auto target_bitrate :
1327 layer.target_bitrate_per_temporal_layer) {
1328 log << target_bitrate.kbps() << ",";
1329 }
1330 log << "]";
1331 }
1332 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1333 }
1334
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001335 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001336 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001337 TestEncoder* test_encoder_;
1338 rtc::Event encoded_frame_event_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001339 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001340 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001341 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001342 uint32_t last_height_ = 0;
1343 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001344 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001345 size_t num_expected_layers_ = 1;
1346 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001347 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001348 int number_of_reconfigurations_ = 0;
1349 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001350 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1351 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001352 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1353 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001354 };
1355
Sergey Silkin5ee69672019-07-02 14:18:34 +02001356 class VideoBitrateAllocatorProxyFactory
1357 : public VideoBitrateAllocatorFactory {
1358 public:
1359 VideoBitrateAllocatorProxyFactory()
1360 : bitrate_allocator_factory_(
1361 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1362
1363 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1364 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001365 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001366 codec_config_ = codec;
1367 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1368 }
1369
1370 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001371 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001372 return codec_config_;
1373 }
1374
1375 private:
1376 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1377
Markus Handella3765182020-07-08 13:13:32 +02001378 mutable Mutex mutex_;
1379 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001380 };
1381
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001382 Clock* clock() { return time_controller_.GetClock(); }
1383 void AdvanceTime(TimeDelta duration) {
1384 time_controller_.AdvanceTime(duration);
1385 }
1386
1387 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1388
1389 protected:
1390 virtual TaskQueueFactory* GetTaskQueueFactory() {
1391 return time_controller_.GetTaskQueueFactory();
1392 }
1393
1394 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001395 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001396 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001397 int codec_width_;
1398 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001399 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001400 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001401 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001402 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001403 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001404 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001405 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001406 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001407};
1408
mflodmancc3d4422017-08-03 08:27:51 -07001409TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001411 DataRate::BitsPerSec(kTargetBitrateBps),
1412 DataRate::BitsPerSec(kTargetBitrateBps),
1413 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001414 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001415 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001416 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001417 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001418 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001419}
1420
mflodmancc3d4422017-08-03 08:27:51 -07001421TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001422 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001423 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001424 // The encoder will cache up to one frame for a short duration. Adding two
1425 // frames means that the first frame will be dropped and the second frame will
1426 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001427 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001428 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001429 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001430 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001431
Henrik Boström381d1092020-05-12 18:49:07 +02001432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001433 DataRate::BitsPerSec(kTargetBitrateBps),
1434 DataRate::BitsPerSec(kTargetBitrateBps),
1435 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001436
Sebastian Janssona3177052018-04-10 13:05:49 +02001437 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001438 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001439 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1440
1441 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001442 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001443}
1444
mflodmancc3d4422017-08-03 08:27:51 -07001445TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001450 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001451 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001452
Henrik Boström381d1092020-05-12 18:49:07 +02001453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1454 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1455 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 // The encoder will cache up to one frame for a short duration. Adding two
1457 // frames means that the first frame will be dropped and the second frame will
1458 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001459 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001460 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001461
Henrik Boström381d1092020-05-12 18:49:07 +02001462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001463 DataRate::BitsPerSec(kTargetBitrateBps),
1464 DataRate::BitsPerSec(kTargetBitrateBps),
1465 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001466 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001467 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1468 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001469 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001470}
1471
mflodmancc3d4422017-08-03 08:27:51 -07001472TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001474 DataRate::BitsPerSec(kTargetBitrateBps),
1475 DataRate::BitsPerSec(kTargetBitrateBps),
1476 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001477 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001478 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001479
1480 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001481 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001482
perkja49cbd32016-09-16 07:53:41 -07001483 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001484 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001485 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001486}
1487
mflodmancc3d4422017-08-03 08:27:51 -07001488TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001490 DataRate::BitsPerSec(kTargetBitrateBps),
1491 DataRate::BitsPerSec(kTargetBitrateBps),
1492 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001493
perkja49cbd32016-09-16 07:53:41 -07001494 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001495 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001496
mflodmancc3d4422017-08-03 08:27:51 -07001497 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001498 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001499 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001500 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1501 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001502}
1503
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001504class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1505 public:
1506 VideoStreamEncoderBlockedTest() {}
1507
1508 TaskQueueFactory* GetTaskQueueFactory() override {
1509 return task_queue_factory_.get();
1510 }
1511
1512 private:
1513 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1514 CreateDefaultTaskQueueFactory();
1515};
1516
1517TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001518 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001519 DataRate::BitsPerSec(kTargetBitrateBps),
1520 DataRate::BitsPerSec(kTargetBitrateBps),
1521 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001522
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001523 int dropped_count = 0;
1524 stats_proxy_->SetDroppedFrameCallback(
1525 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1526 ++dropped_count;
1527 });
1528
perkj26091b12016-09-01 01:17:40 -07001529 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001530 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001531 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001532 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1533 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001534 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1535 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001536 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001537 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001538
mflodmancc3d4422017-08-03 08:27:51 -07001539 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001540
1541 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001542}
1543
Noah Richards51db4212019-06-12 06:59:12 -07001544TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420Conversion) {
Henrik Boström381d1092020-05-12 18:49:07 +02001545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001546 DataRate::BitsPerSec(kTargetBitrateBps),
1547 DataRate::BitsPerSec(kTargetBitrateBps),
1548 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001549
1550 rtc::Event frame_destroyed_event;
1551 video_source_.IncomingCapturedFrame(
1552 CreateFakeNativeFrame(1, &frame_destroyed_event));
1553 ExpectDroppedFrame();
1554 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1555 video_stream_encoder_->Stop();
1556}
1557
1558TEST_F(VideoStreamEncoderTest, DropFrameWithFailedI420ConversionWithCrop) {
1559 // Use the cropping factory.
1560 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001561 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001562 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1563 kMaxPayloadLength);
1564 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1565
1566 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001568 DataRate::BitsPerSec(kTargetBitrateBps),
1569 DataRate::BitsPerSec(kTargetBitrateBps),
1570 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001571 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1572 WaitForEncodedFrame(1);
1573 // The encoder will have been configured once.
1574 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1575 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1576 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1577
1578 // Now send in a fake frame that needs to be cropped as the width/height
1579 // aren't divisible by 4 (see CreateEncoderStreams above).
1580 rtc::Event frame_destroyed_event;
1581 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1582 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
1583 ExpectDroppedFrame();
1584 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
1585 video_stream_encoder_->Stop();
1586}
1587
Evan Shrubsole895556e2020-10-05 09:15:13 +02001588TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1589 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1590 DataRate::BitsPerSec(kTargetBitrateBps),
1591 DataRate::BitsPerSec(kTargetBitrateBps),
1592 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1593
1594 video_source_.IncomingCapturedFrame(
1595 CreateNV12Frame(1, codec_width_, codec_height_));
1596 WaitForEncodedFrame(1);
1597 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1598 fake_encoder_.GetLastInputPixelFormat());
1599 video_stream_encoder_->Stop();
1600}
1601
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001602TEST_F(VideoStreamEncoderTest,
1603 NativeFrameIsConvertedToI420IfNoFrameTypePreference) {
1604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1605 DataRate::BitsPerSec(kTargetBitrateBps),
1606 DataRate::BitsPerSec(kTargetBitrateBps),
1607 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1608
1609 fake_encoder_.SetPreferredPixelFormats({});
1610
1611 rtc::Event frame_destroyed_event;
1612 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1613 1, &frame_destroyed_event, codec_width_, codec_height_));
1614 WaitForEncodedFrame(1);
1615 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1616 fake_encoder_.GetLastInputPixelFormat());
1617 video_stream_encoder_->Stop();
1618}
1619
1620TEST_F(VideoStreamEncoderTest, NativeFrameMappedToPreferredPixelFormat) {
1621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1622 DataRate::BitsPerSec(kTargetBitrateBps),
1623 DataRate::BitsPerSec(kTargetBitrateBps),
1624 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1625
1626 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1627
1628 rtc::Event frame_destroyed_event;
1629 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1630 1, &frame_destroyed_event, codec_width_, codec_height_));
1631 WaitForEncodedFrame(1);
1632 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1633 fake_encoder_.GetLastInputPixelFormat());
1634 video_stream_encoder_->Stop();
1635}
1636
1637TEST_F(VideoStreamEncoderTest, NativeFrameConvertedToI420IfMappingNotFeasible) {
1638 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1639 DataRate::BitsPerSec(kTargetBitrateBps),
1640 DataRate::BitsPerSec(kTargetBitrateBps),
1641 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1642
1643 // Fake NV12 native frame does not allow mapping to I444.
1644 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1645
1646 rtc::Event frame_destroyed_event;
1647 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1648 1, &frame_destroyed_event, codec_width_, codec_height_));
1649 WaitForEncodedFrame(1);
1650 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1651 fake_encoder_.GetLastInputPixelFormat());
1652 video_stream_encoder_->Stop();
1653}
1654
Evan Shrubsole895556e2020-10-05 09:15:13 +02001655TEST_F(VideoStreamEncoderTest, NativeFrameBackedByNV12FrameIsEncodedFromI420) {
1656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1657 DataRate::BitsPerSec(kTargetBitrateBps),
1658 DataRate::BitsPerSec(kTargetBitrateBps),
1659 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1660
1661 rtc::Event frame_destroyed_event;
1662 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1663 1, &frame_destroyed_event, codec_width_, codec_height_));
1664 WaitForEncodedFrame(1);
1665 EXPECT_EQ(VideoFrameBuffer::Type::kI420,
1666 fake_encoder_.GetLastInputPixelFormat());
1667 video_stream_encoder_->Stop();
1668}
1669
Ying Wang9b881ab2020-02-07 14:29:32 +01001670TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001672 DataRate::BitsPerSec(kTargetBitrateBps),
1673 DataRate::BitsPerSec(kTargetBitrateBps),
1674 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001675 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1676 WaitForEncodedFrame(1);
1677
Henrik Boström381d1092020-05-12 18:49:07 +02001678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001679 DataRate::BitsPerSec(kTargetBitrateBps),
1680 DataRate::BitsPerSec(kTargetBitrateBps),
1681 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001682 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1683 // frames. Adding two frames means that the first frame will be dropped and
1684 // the second frame will be sent to the encoder.
1685 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1686 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1687 WaitForEncodedFrame(3);
1688 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1689 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1690 WaitForEncodedFrame(5);
1691 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1692 video_stream_encoder_->Stop();
1693}
1694
mflodmancc3d4422017-08-03 08:27:51 -07001695TEST_F(VideoStreamEncoderTest,
1696 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001698 DataRate::BitsPerSec(kTargetBitrateBps),
1699 DataRate::BitsPerSec(kTargetBitrateBps),
1700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001701 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001702
1703 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001704 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001705 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001706 // The encoder will have been configured once when the first frame is
1707 // received.
1708 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001709
1710 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001711 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001712 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001713 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001714 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001715
1716 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001717 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001718 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001719 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001720 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001721
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001723}
1724
mflodmancc3d4422017-08-03 08:27:51 -07001725TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001727 DataRate::BitsPerSec(kTargetBitrateBps),
1728 DataRate::BitsPerSec(kTargetBitrateBps),
1729 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001730
1731 // Capture a frame and wait for it to synchronize with the encoder thread.
1732 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001733 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001734 // The encoder will have been configured once.
1735 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001736 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1737 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1738
1739 codec_width_ *= 2;
1740 codec_height_ *= 2;
1741 // Capture a frame with a higher resolution and wait for it to synchronize
1742 // with the encoder thread.
1743 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001744 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001745 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1746 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001747 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001748
mflodmancc3d4422017-08-03 08:27:51 -07001749 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001750}
1751
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001752TEST_F(VideoStreamEncoderTest,
1753 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001754 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001755 DataRate::BitsPerSec(kTargetBitrateBps),
1756 DataRate::BitsPerSec(kTargetBitrateBps),
1757 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001758
1759 // Capture a frame and wait for it to synchronize with the encoder thread.
1760 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1761 WaitForEncodedFrame(1);
1762
1763 VideoEncoderConfig video_encoder_config;
1764 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1765 // Changing the max payload data length recreates encoder.
1766 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1767 kMaxPayloadLength / 2);
1768
1769 // Capture a frame and wait for it to synchronize with the encoder thread.
1770 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1771 WaitForEncodedFrame(2);
1772 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1773
1774 video_stream_encoder_->Stop();
1775}
1776
Sergey Silkin5ee69672019-07-02 14:18:34 +02001777TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001779 DataRate::BitsPerSec(kTargetBitrateBps),
1780 DataRate::BitsPerSec(kTargetBitrateBps),
1781 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001782
1783 VideoEncoderConfig video_encoder_config;
1784 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1785 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1786 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1787 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1788 kMaxPayloadLength);
1789
1790 // Capture a frame and wait for it to synchronize with the encoder thread.
1791 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1792 WaitForEncodedFrame(1);
1793 // The encoder will have been configured once when the first frame is
1794 // received.
1795 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1796 EXPECT_EQ(kTargetBitrateBps,
1797 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1798 EXPECT_EQ(kStartBitrateBps,
1799 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1800
Sergey Silkin6456e352019-07-08 17:56:40 +02001801 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1802 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001803 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1804 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1805 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1806 kMaxPayloadLength);
1807
1808 // Capture a frame and wait for it to synchronize with the encoder thread.
1809 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1810 WaitForEncodedFrame(2);
1811 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1812 // Bitrate limits have changed - rate allocator should be reconfigured,
1813 // encoder should not be reconfigured.
1814 EXPECT_EQ(kTargetBitrateBps * 2,
1815 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1816 EXPECT_EQ(kStartBitrateBps * 2,
1817 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1818 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1819
1820 video_stream_encoder_->Stop();
1821}
1822
Sergey Silkin6456e352019-07-08 17:56:40 +02001823TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001824 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001826 DataRate::BitsPerSec(kTargetBitrateBps),
1827 DataRate::BitsPerSec(kTargetBitrateBps),
1828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001829
Sergey Silkincd02eba2020-01-20 14:48:40 +01001830 const uint32_t kMinEncBitrateKbps = 100;
1831 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001832 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001833 /*frame_size_pixels=*/codec_width_ * codec_height_,
1834 /*min_start_bitrate_bps=*/0,
1835 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1836 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001837 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1838
Sergey Silkincd02eba2020-01-20 14:48:40 +01001839 VideoEncoderConfig video_encoder_config;
1840 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1841 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1842 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1843 (kMinEncBitrateKbps + 1) * 1000;
1844 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1845 kMaxPayloadLength);
1846
1847 // When both encoder and app provide bitrate limits, the intersection of
1848 // provided sets should be used.
1849 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1850 WaitForEncodedFrame(1);
1851 EXPECT_EQ(kMaxEncBitrateKbps,
1852 bitrate_allocator_factory_.codec_config().maxBitrate);
1853 EXPECT_EQ(kMinEncBitrateKbps + 1,
1854 bitrate_allocator_factory_.codec_config().minBitrate);
1855
1856 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1857 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1858 (kMinEncBitrateKbps - 1) * 1000;
1859 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1860 kMaxPayloadLength);
1861 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001862 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001863 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001864 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001865 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001866 bitrate_allocator_factory_.codec_config().minBitrate);
1867
Sergey Silkincd02eba2020-01-20 14:48:40 +01001868 video_stream_encoder_->Stop();
1869}
1870
1871TEST_F(VideoStreamEncoderTest,
1872 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001874 DataRate::BitsPerSec(kTargetBitrateBps),
1875 DataRate::BitsPerSec(kTargetBitrateBps),
1876 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001877
1878 const uint32_t kMinAppBitrateKbps = 100;
1879 const uint32_t kMaxAppBitrateKbps = 200;
1880 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1881 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1882 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1883 /*frame_size_pixels=*/codec_width_ * codec_height_,
1884 /*min_start_bitrate_bps=*/0,
1885 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1886 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1887 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1888
1889 VideoEncoderConfig video_encoder_config;
1890 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1891 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1892 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1893 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001894 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1895 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001896
Sergey Silkincd02eba2020-01-20 14:48:40 +01001897 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1898 WaitForEncodedFrame(1);
1899 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001901 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001902 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001903
1904 video_stream_encoder_->Stop();
1905}
1906
1907TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001908 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001910 DataRate::BitsPerSec(kTargetBitrateBps),
1911 DataRate::BitsPerSec(kTargetBitrateBps),
1912 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001913
1914 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001915 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001916 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001917 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001918 fake_encoder_.SetResolutionBitrateLimits(
1919 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1920
1921 VideoEncoderConfig video_encoder_config;
1922 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1923 video_encoder_config.max_bitrate_bps = 0;
1924 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1925 kMaxPayloadLength);
1926
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001927 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001928 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1929 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001930 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1931 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001932 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1933 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1934
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001935 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001936 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1937 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001938 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1939 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001940 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1941 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1942
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001943 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001944 // encoder for 360p should be used.
1945 video_source_.IncomingCapturedFrame(
1946 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1947 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001948 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1949 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001950 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1951 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1952
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001953 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001954 // ignored.
1955 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1956 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001957 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1958 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001959 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1960 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001961 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1962 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001963 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1964 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1965
1966 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1967 // for 270p should be used.
1968 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1969 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001970 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1971 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001972 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1973 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1974
1975 video_stream_encoder_->Stop();
1976}
1977
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001978TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001980 DataRate::BitsPerSec(kTargetBitrateBps),
1981 DataRate::BitsPerSec(kTargetBitrateBps),
1982 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001983
1984 VideoEncoderConfig video_encoder_config;
1985 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1986 video_encoder_config.max_bitrate_bps = 0;
1987 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1988 kMaxPayloadLength);
1989
1990 // Encode 720p frame to get the default encoder target bitrate.
1991 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1992 WaitForEncodedFrame(1);
1993 const uint32_t kDefaultTargetBitrateFor720pKbps =
1994 bitrate_allocator_factory_.codec_config()
1995 .simulcastStream[0]
1996 .targetBitrate;
1997
1998 // Set the max recommended encoder bitrate to something lower than the default
1999 // target bitrate.
2000 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2001 1280 * 720, 10 * 1000, 10 * 1000,
2002 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2003 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2004
2005 // Change resolution to trigger encoder reinitialization.
2006 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2007 WaitForEncodedFrame(2);
2008 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2009 WaitForEncodedFrame(3);
2010
2011 // Ensure the target bitrate is capped by the max bitrate.
2012 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2013 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2014 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2015 .simulcastStream[0]
2016 .targetBitrate *
2017 1000,
2018 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2019
2020 video_stream_encoder_->Stop();
2021}
2022
Åsa Perssona7e34d32021-01-20 15:36:13 +01002023TEST_F(VideoStreamEncoderTest,
2024 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2025 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2026 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2027 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2028 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2029 fake_encoder_.SetResolutionBitrateLimits(
2030 {kEncoderLimits270p, kEncoderLimits360p});
2031
2032 // Two streams, highest stream active.
2033 VideoEncoderConfig config;
2034 const int kNumStreams = 2;
2035 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2036 config.max_bitrate_bps = 0;
2037 config.simulcast_layers[0].active = false;
2038 config.simulcast_layers[1].active = true;
2039 config.video_stream_factory =
2040 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2041 "VP8", /*max qp*/ 56, /*screencast*/ false,
2042 /*screenshare enabled*/ false);
2043 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2044
2045 // The encoder bitrate limits for 270p should be used.
2046 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2047 EXPECT_FALSE(WaitForFrame(1000));
2048 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2049 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2050 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2051 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2052 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2053
2054 // The encoder bitrate limits for 360p should be used.
2055 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2056 EXPECT_FALSE(WaitForFrame(1000));
2057 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2058 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2059 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2060 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2061
2062 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2063 video_source_.IncomingCapturedFrame(
2064 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2065 EXPECT_FALSE(WaitForFrame(1000));
2066 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2067 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2068 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2069 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2070
2071 // Resolution higher than 360p. Encoder limits should be ignored.
2072 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2073 EXPECT_FALSE(WaitForFrame(1000));
2074 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2075 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2076 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2077 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2078 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2079 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2080 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2081 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2082
2083 // Resolution lower than 270p. The encoder limits for 270p should be used.
2084 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2085 EXPECT_FALSE(WaitForFrame(1000));
2086 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2087 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2088 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2089 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2090
2091 video_stream_encoder_->Stop();
2092}
2093
2094TEST_F(VideoStreamEncoderTest,
2095 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2096 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2097 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2098 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2099 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2100 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2101 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2102 fake_encoder_.SetResolutionBitrateLimits(
2103 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2104
2105 // Three streams, middle stream active.
2106 VideoEncoderConfig config;
2107 const int kNumStreams = 3;
2108 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2109 config.simulcast_layers[0].active = false;
2110 config.simulcast_layers[1].active = true;
2111 config.simulcast_layers[2].active = false;
2112 config.video_stream_factory =
2113 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2114 "VP8", /*max qp*/ 56, /*screencast*/ false,
2115 /*screenshare enabled*/ false);
2116 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2117
2118 // The encoder bitrate limits for 360p should be used.
2119 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2120 EXPECT_FALSE(WaitForFrame(1000));
2121 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2122 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2123 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2124 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2125 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2126
2127 // The encoder bitrate limits for 270p should be used.
2128 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2129 EXPECT_FALSE(WaitForFrame(1000));
2130 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2131 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2132 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2133 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2134
2135 video_stream_encoder_->Stop();
2136}
2137
2138TEST_F(VideoStreamEncoderTest,
2139 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2140 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2141 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2142 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2143 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2144 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2145 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2146 fake_encoder_.SetResolutionBitrateLimits(
2147 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2148
2149 // Three streams, lowest stream active.
2150 VideoEncoderConfig config;
2151 const int kNumStreams = 3;
2152 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2153 config.simulcast_layers[0].active = true;
2154 config.simulcast_layers[1].active = false;
2155 config.simulcast_layers[2].active = false;
2156 config.video_stream_factory =
2157 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2158 "VP8", /*max qp*/ 56, /*screencast*/ false,
2159 /*screenshare enabled*/ false);
2160 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2161
2162 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2163 // on lowest stream, limits for 270p should not be used
2164 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2165 EXPECT_FALSE(WaitForFrame(1000));
2166 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2167 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2168 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2169 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2170 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2171
2172 video_stream_encoder_->Stop();
2173}
2174
2175TEST_F(VideoStreamEncoderTest,
2176 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2177 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2178 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2179 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2180 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2181 fake_encoder_.SetResolutionBitrateLimits(
2182 {kEncoderLimits270p, kEncoderLimits360p});
2183 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2184
2185 // Two streams, highest stream active.
2186 VideoEncoderConfig config;
2187 const int kNumStreams = 2;
2188 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2189 config.simulcast_layers[0].active = false;
2190 config.simulcast_layers[1].active = true;
2191 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2192 config.video_stream_factory =
2193 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2194 "VP8", /*max qp*/ 56, /*screencast*/ false,
2195 /*screenshare enabled*/ false);
2196 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2197
2198 // The encoder bitrate limits for 270p should be used.
2199 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2200 EXPECT_FALSE(WaitForFrame(1000));
2201 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2202 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2203 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2204 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2205 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2206
2207 // The max configured bitrate is less than the encoder limit for 360p.
2208 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2209 EXPECT_FALSE(WaitForFrame(1000));
2210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2211 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2212 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2213 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2214
2215 video_stream_encoder_->Stop();
2216}
2217
mflodmancc3d4422017-08-03 08:27:51 -07002218TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002219 EXPECT_TRUE(video_source_.has_sinks());
2220 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002221 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002222 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002223 EXPECT_FALSE(video_source_.has_sinks());
2224 EXPECT_TRUE(new_video_source.has_sinks());
2225
mflodmancc3d4422017-08-03 08:27:51 -07002226 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002227}
2228
mflodmancc3d4422017-08-03 08:27:51 -07002229TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002230 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002231 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002232 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002233 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002234}
2235
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002236class ResolutionAlignmentTest
2237 : public VideoStreamEncoderTest,
2238 public ::testing::WithParamInterface<
2239 ::testing::tuple<int, std::vector<double>>> {
2240 public:
2241 ResolutionAlignmentTest()
2242 : requested_alignment_(::testing::get<0>(GetParam())),
2243 scale_factors_(::testing::get<1>(GetParam())) {}
2244
2245 protected:
2246 const int requested_alignment_;
2247 const std::vector<double> scale_factors_;
2248};
2249
2250INSTANTIATE_TEST_SUITE_P(
2251 AlignmentAndScaleFactors,
2252 ResolutionAlignmentTest,
2253 ::testing::Combine(
2254 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2255 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2256 std::vector<double>{-1.0, -1.0},
2257 std::vector<double>{-1.0, -1.0, -1.0},
2258 std::vector<double>{4.0, 2.0, 1.0},
2259 std::vector<double>{9999.0, -1.0, 1.0},
2260 std::vector<double>{3.99, 2.01, 1.0},
2261 std::vector<double>{4.9, 1.7, 1.25},
2262 std::vector<double>{10.0, 4.0, 3.0},
2263 std::vector<double>{1.75, 3.5},
2264 std::vector<double>{1.5, 2.5},
2265 std::vector<double>{1.3, 1.0})));
2266
2267TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2268 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002269 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002270 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2271 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2272
2273 // Fill config with the scaling factor by which to reduce encoding size.
2274 const int num_streams = scale_factors_.size();
2275 VideoEncoderConfig config;
2276 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2277 for (int i = 0; i < num_streams; ++i) {
2278 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2279 }
2280 config.video_stream_factory =
2281 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2282 "VP8", /*max qp*/ 56, /*screencast*/ false,
2283 /*screenshare enabled*/ false);
2284 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2285
Henrik Boström381d1092020-05-12 18:49:07 +02002286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002287 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2288 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2289 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2290 // Wait for all layers before triggering event.
2291 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002292
2293 // On the 1st frame, we should have initialized the encoder and
2294 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002295 int64_t timestamp_ms = kFrameIntervalMs;
2296 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2297 WaitForEncodedFrame(timestamp_ms);
2298 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002299
2300 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2301 // (It's up the to the encoder to potentially drop the previous frame,
2302 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002303 timestamp_ms += kFrameIntervalMs;
2304 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2305 WaitForEncodedFrame(timestamp_ms);
2306 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2307
2308 VideoCodec codec = fake_encoder_.video_codec();
2309 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2310 // Frame size should be a multiple of the requested alignment.
2311 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2312 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2313 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2314 // Aspect ratio should match.
2315 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2316 codec.height * codec.simulcastStream[i].width);
2317 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002318
2319 video_stream_encoder_->Stop();
2320}
2321
Jonathan Yubc771b72017-12-08 17:04:29 -08002322TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2323 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002324 const int kWidth = 1280;
2325 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002326
2327 // We rely on the automatic resolution adaptation, but we handle framerate
2328 // adaptation manually by mocking the stats proxy.
2329 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002331 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002333 DataRate::BitsPerSec(kTargetBitrateBps),
2334 DataRate::BitsPerSec(kTargetBitrateBps),
2335 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002336 video_stream_encoder_->SetSource(&video_source_,
2337 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002338 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002340 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002341 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2342
Jonathan Yubc771b72017-12-08 17:04:29 -08002343 // Adapt down as far as possible.
2344 rtc::VideoSinkWants last_wants;
2345 int64_t t = 1;
2346 int loop_count = 0;
2347 do {
2348 ++loop_count;
2349 last_wants = video_source_.sink_wants();
2350
2351 // Simulate the framerate we've been asked to adapt to.
2352 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2353 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2354 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2355 mock_stats.input_frame_rate = fps;
2356 stats_proxy_->SetMockStats(mock_stats);
2357
2358 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2359 sink_.WaitForEncodedFrame(t);
2360 t += frame_interval_ms;
2361
mflodmancc3d4422017-08-03 08:27:51 -07002362 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002363 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002364 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002365 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2366 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002367 } while (video_source_.sink_wants().max_pixel_count <
2368 last_wants.max_pixel_count ||
2369 video_source_.sink_wants().max_framerate_fps <
2370 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002371
Jonathan Yubc771b72017-12-08 17:04:29 -08002372 // Verify that we've adapted all the way down.
2373 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002374 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002375 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2376 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002377 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002378 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2379 *video_source_.last_sent_height());
2380 EXPECT_EQ(kMinBalancedFramerateFps,
2381 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002382
Jonathan Yubc771b72017-12-08 17:04:29 -08002383 // Adapt back up the same number of times we adapted down.
2384 for (int i = 0; i < loop_count - 1; ++i) {
2385 last_wants = video_source_.sink_wants();
2386
2387 // Simulate the framerate we've been asked to adapt to.
2388 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2389 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2390 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2391 mock_stats.input_frame_rate = fps;
2392 stats_proxy_->SetMockStats(mock_stats);
2393
2394 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2395 sink_.WaitForEncodedFrame(t);
2396 t += frame_interval_ms;
2397
Henrik Boström91aa7322020-04-28 12:24:33 +02002398 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002399 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002400 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002401 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2402 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002403 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2404 last_wants.max_pixel_count ||
2405 video_source_.sink_wants().max_framerate_fps >
2406 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002407 }
2408
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002409 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002410 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002411 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002412 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2413 EXPECT_EQ((loop_count - 1) * 2,
2414 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002415
mflodmancc3d4422017-08-03 08:27:51 -07002416 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002417}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002418
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002419TEST_F(VideoStreamEncoderTest,
2420 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2421 video_stream_encoder_->OnBitrateUpdated(
2422 DataRate::BitsPerSec(kTargetBitrateBps),
2423 DataRate::BitsPerSec(kTargetBitrateBps),
2424 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002425 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002426
2427 const int kFrameWidth = 1280;
2428 const int kFrameHeight = 720;
2429
2430 int64_t ntp_time = kFrameIntervalMs;
2431
2432 // Force an input frame rate to be available, or the adaptation call won't
2433 // know what framerate to adapt form.
2434 const int kInputFps = 30;
2435 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2436 stats.input_frame_rate = kInputFps;
2437 stats_proxy_->SetMockStats(stats);
2438
2439 video_source_.set_adaptation_enabled(true);
2440 video_stream_encoder_->SetSource(
2441 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002442 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002443 video_source_.IncomingCapturedFrame(
2444 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2445 sink_.WaitForEncodedFrame(ntp_time);
2446 ntp_time += kFrameIntervalMs;
2447
2448 // Trigger CPU overuse.
2449 video_stream_encoder_->TriggerCpuOveruse();
2450 video_source_.IncomingCapturedFrame(
2451 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2452 sink_.WaitForEncodedFrame(ntp_time);
2453 ntp_time += kFrameIntervalMs;
2454
2455 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2456 EXPECT_EQ(std::numeric_limits<int>::max(),
2457 video_source_.sink_wants().max_pixel_count);
2458 // Some framerate constraint should be set.
2459 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2460 EXPECT_LT(restricted_fps, kInputFps);
2461 video_source_.IncomingCapturedFrame(
2462 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2463 sink_.WaitForEncodedFrame(ntp_time);
2464 ntp_time += 100;
2465
Henrik Boström2671dac2020-05-19 16:29:09 +02002466 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002467 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2468 // Give the encoder queue time to process the change in degradation preference
2469 // by waiting for an encoded frame.
2470 video_source_.IncomingCapturedFrame(
2471 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2472 sink_.WaitForEncodedFrame(ntp_time);
2473 ntp_time += kFrameIntervalMs;
2474
2475 video_stream_encoder_->TriggerQualityLow();
2476 video_source_.IncomingCapturedFrame(
2477 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2478 sink_.WaitForEncodedFrame(ntp_time);
2479 ntp_time += kFrameIntervalMs;
2480
2481 // Some resolution constraint should be set.
2482 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2483 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2484 kFrameWidth * kFrameHeight);
2485 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2486
2487 int pixel_count = video_source_.sink_wants().max_pixel_count;
2488 // Triggering a CPU underuse should not change the sink wants since it has
2489 // not been overused for resolution since we changed degradation preference.
2490 video_stream_encoder_->TriggerCpuUnderuse();
2491 video_source_.IncomingCapturedFrame(
2492 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2493 sink_.WaitForEncodedFrame(ntp_time);
2494 ntp_time += kFrameIntervalMs;
2495 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2496 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2497
Evan Shrubsole64469032020-06-11 10:45:29 +02002498 // Change the degradation preference back. CPU underuse should not adapt since
2499 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002500 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002501 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2502 video_source_.IncomingCapturedFrame(
2503 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2504 sink_.WaitForEncodedFrame(ntp_time);
2505 ntp_time += 100;
2506 // Resolution adaptations is gone after changing degradation preference.
2507 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2508 EXPECT_EQ(std::numeric_limits<int>::max(),
2509 video_source_.sink_wants().max_pixel_count);
2510 // The fps adaptation from above is now back.
2511 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2512
2513 // Trigger CPU underuse.
2514 video_stream_encoder_->TriggerCpuUnderuse();
2515 video_source_.IncomingCapturedFrame(
2516 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2517 sink_.WaitForEncodedFrame(ntp_time);
2518 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002519 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2520
2521 // Trigger QP underuse, fps should return to normal.
2522 video_stream_encoder_->TriggerQualityHigh();
2523 video_source_.IncomingCapturedFrame(
2524 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2525 sink_.WaitForEncodedFrame(ntp_time);
2526 ntp_time += kFrameIntervalMs;
2527 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002528
2529 video_stream_encoder_->Stop();
2530}
2531
mflodmancc3d4422017-08-03 08:27:51 -07002532TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps),
2536 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002537 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002538
sprangc5d62e22017-04-02 23:53:04 -07002539 const int kFrameWidth = 1280;
2540 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002541
Åsa Persson8c1bf952018-09-13 10:42:19 +02002542 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002543
kthelgason5e13d412016-12-01 03:59:51 -08002544 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002545 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002546 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002547 frame_timestamp += kFrameIntervalMs;
2548
perkj803d97f2016-11-01 11:45:46 -07002549 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002550 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002551 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002552 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002553 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002554 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002555
asapersson0944a802017-04-07 00:57:58 -07002556 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002557 // wanted resolution.
2558 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2559 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2560 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002561 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002562
2563 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002564 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002565 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002566 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002567 // Give the encoder queue time to process the change in degradation preference
2568 // by waiting for an encoded frame.
2569 new_video_source.IncomingCapturedFrame(
2570 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2571 sink_.WaitForEncodedFrame(frame_timestamp);
2572 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002573 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002574 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002575
sprangc5d62e22017-04-02 23:53:04 -07002576 // Force an input frame rate to be available, or the adaptation call won't
2577 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002578 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002579 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002580 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002581 stats_proxy_->SetMockStats(stats);
2582
mflodmancc3d4422017-08-03 08:27:51 -07002583 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002584 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002585 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002586 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002587 frame_timestamp += kFrameIntervalMs;
2588
2589 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002590 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002591 EXPECT_EQ(std::numeric_limits<int>::max(),
2592 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002593 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002594
asapersson02465b82017-04-10 01:12:52 -07002595 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002596 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2597 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002598 // Give the encoder queue time to process the change in degradation preference
2599 // by waiting for an encoded frame.
2600 new_video_source.IncomingCapturedFrame(
2601 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2602 sink_.WaitForEncodedFrame(frame_timestamp);
2603 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002604 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002605
mflodmancc3d4422017-08-03 08:27:51 -07002606 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002607 new_video_source.IncomingCapturedFrame(
2608 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002609 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002610 frame_timestamp += kFrameIntervalMs;
2611
2612 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002613 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002614
2615 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002616 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002617 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002618 // Give the encoder queue time to process the change in degradation preference
2619 // by waiting for an encoded frame.
2620 new_video_source.IncomingCapturedFrame(
2621 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2622 sink_.WaitForEncodedFrame(frame_timestamp);
2623 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002624 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2625 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002626 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002627 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002628
2629 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002630 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002631 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002632 // Give the encoder queue time to process the change in degradation preference
2633 // by waiting for an encoded frame.
2634 new_video_source.IncomingCapturedFrame(
2635 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2636 sink_.WaitForEncodedFrame(frame_timestamp);
2637 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002638 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2639 EXPECT_EQ(std::numeric_limits<int>::max(),
2640 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002641 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002642
mflodmancc3d4422017-08-03 08:27:51 -07002643 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002644}
2645
mflodmancc3d4422017-08-03 08:27:51 -07002646TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002648 DataRate::BitsPerSec(kTargetBitrateBps),
2649 DataRate::BitsPerSec(kTargetBitrateBps),
2650 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002651
asaperssonfab67072017-04-04 05:51:49 -07002652 const int kWidth = 1280;
2653 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002654 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002655 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002656 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2657 EXPECT_FALSE(stats.bw_limited_resolution);
2658 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2659
2660 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002661 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002662 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002663 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002664
2665 stats = stats_proxy_->GetStats();
2666 EXPECT_TRUE(stats.bw_limited_resolution);
2667 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2668
2669 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002670 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002671 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002673
2674 stats = stats_proxy_->GetStats();
2675 EXPECT_FALSE(stats.bw_limited_resolution);
2676 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2677 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2678
mflodmancc3d4422017-08-03 08:27:51 -07002679 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002680}
2681
mflodmancc3d4422017-08-03 08:27:51 -07002682TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002684 DataRate::BitsPerSec(kTargetBitrateBps),
2685 DataRate::BitsPerSec(kTargetBitrateBps),
2686 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002687
2688 const int kWidth = 1280;
2689 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002690 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002691 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002692 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2693 EXPECT_FALSE(stats.cpu_limited_resolution);
2694 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2695
2696 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002697 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002698 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002699 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002700
2701 stats = stats_proxy_->GetStats();
2702 EXPECT_TRUE(stats.cpu_limited_resolution);
2703 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2704
2705 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002706 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002707 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002708 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002709
2710 stats = stats_proxy_->GetStats();
2711 EXPECT_FALSE(stats.cpu_limited_resolution);
2712 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002713 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002714
mflodmancc3d4422017-08-03 08:27:51 -07002715 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002716}
2717
mflodmancc3d4422017-08-03 08:27:51 -07002718TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002720 DataRate::BitsPerSec(kTargetBitrateBps),
2721 DataRate::BitsPerSec(kTargetBitrateBps),
2722 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002723
asaperssonfab67072017-04-04 05:51:49 -07002724 const int kWidth = 1280;
2725 const int kHeight = 720;
2726 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002727 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002728 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002729 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002730 EXPECT_FALSE(stats.cpu_limited_resolution);
2731 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2732
asaperssonfab67072017-04-04 05:51:49 -07002733 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002734 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002735 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002736 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002737 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002738 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002739 EXPECT_TRUE(stats.cpu_limited_resolution);
2740 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2741
2742 // Set new source with adaptation still enabled.
2743 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002744 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002745 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002746
asaperssonfab67072017-04-04 05:51:49 -07002747 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002748 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002749 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002750 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002751 EXPECT_TRUE(stats.cpu_limited_resolution);
2752 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2753
2754 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002755 video_stream_encoder_->SetSource(&new_video_source,
2756 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002757
asaperssonfab67072017-04-04 05:51:49 -07002758 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002759 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002760 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002761 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002762 EXPECT_FALSE(stats.cpu_limited_resolution);
2763 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2764
2765 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002766 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002767 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002768
asaperssonfab67072017-04-04 05:51:49 -07002769 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002771 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002772 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002773 EXPECT_TRUE(stats.cpu_limited_resolution);
2774 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2775
asaperssonfab67072017-04-04 05:51:49 -07002776 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002777 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002778 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002779 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002780 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002781 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002782 EXPECT_FALSE(stats.cpu_limited_resolution);
2783 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002784 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002785
mflodmancc3d4422017-08-03 08:27:51 -07002786 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002787}
2788
mflodmancc3d4422017-08-03 08:27:51 -07002789TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002791 DataRate::BitsPerSec(kTargetBitrateBps),
2792 DataRate::BitsPerSec(kTargetBitrateBps),
2793 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002794
asaperssonfab67072017-04-04 05:51:49 -07002795 const int kWidth = 1280;
2796 const int kHeight = 720;
2797 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002798 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002799 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002800 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002801 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002802 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002803
2804 // Set new source with adaptation still enabled.
2805 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002806 video_stream_encoder_->SetSource(&new_video_source,
2807 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002808
asaperssonfab67072017-04-04 05:51:49 -07002809 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002810 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002811 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002812 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002813 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002814 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002815
asaperssonfab67072017-04-04 05:51:49 -07002816 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002817 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002818 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002820 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002821 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002822 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002823 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002824
asaperssonfab67072017-04-04 05:51:49 -07002825 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002826 video_stream_encoder_->SetSource(&new_video_source,
2827 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002828
asaperssonfab67072017-04-04 05:51:49 -07002829 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002830 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002831 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002832 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002833 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002834 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002835
asapersson02465b82017-04-10 01:12:52 -07002836 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002838 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002839
asaperssonfab67072017-04-04 05:51:49 -07002840 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002841 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002842 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002843 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002844 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002845 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2846 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002847
mflodmancc3d4422017-08-03 08:27:51 -07002848 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002849}
2850
mflodmancc3d4422017-08-03 08:27:51 -07002851TEST_F(VideoStreamEncoderTest,
2852 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002854 DataRate::BitsPerSec(kTargetBitrateBps),
2855 DataRate::BitsPerSec(kTargetBitrateBps),
2856 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002857
2858 const int kWidth = 1280;
2859 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002860 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002861 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002862 video_source_.IncomingCapturedFrame(
2863 CreateFrame(timestamp_ms, kWidth, kHeight));
2864 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2867 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2868
2869 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002870 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002871 timestamp_ms += kFrameIntervalMs;
2872 video_source_.IncomingCapturedFrame(
2873 CreateFrame(timestamp_ms, kWidth, kHeight));
2874 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002875 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2877 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2878
2879 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002880 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002881 timestamp_ms += kFrameIntervalMs;
2882 video_source_.IncomingCapturedFrame(
2883 CreateFrame(timestamp_ms, kWidth, kHeight));
2884 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002885 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2886 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2887 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2888
Niels Möller4db138e2018-04-19 09:04:13 +02002889 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002890 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002891
2892 VideoEncoderConfig video_encoder_config;
2893 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2894 // Make format different, to force recreation of encoder.
2895 video_encoder_config.video_format.parameters["foo"] = "foo";
2896 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002897 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002898 timestamp_ms += kFrameIntervalMs;
2899 video_source_.IncomingCapturedFrame(
2900 CreateFrame(timestamp_ms, kWidth, kHeight));
2901 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002902 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2904 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2905
mflodmancc3d4422017-08-03 08:27:51 -07002906 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002907}
2908
mflodmancc3d4422017-08-03 08:27:51 -07002909TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002910 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002911 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps),
2914 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
2915
2916 const int kWidth = 1280;
2917 const int kHeight = 720;
2918 int sequence = 1;
2919
2920 // Enable BALANCED preference, no initial limitation.
2921 test::FrameForwarder source;
2922 video_stream_encoder_->SetSource(&source,
2923 webrtc::DegradationPreference::BALANCED);
2924 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2925 WaitForEncodedFrame(sequence++);
2926 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2927 EXPECT_FALSE(stats.cpu_limited_resolution);
2928 EXPECT_FALSE(stats.cpu_limited_framerate);
2929 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2930
2931 // Trigger CPU overuse, should now adapt down.
2932 video_stream_encoder_->TriggerCpuOveruse();
2933 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2934 WaitForEncodedFrame(sequence++);
2935 stats = stats_proxy_->GetStats();
2936 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2937
2938 // Set new degradation preference should clear restrictions since we changed
2939 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002940 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002941 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2942 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2943 WaitForEncodedFrame(sequence++);
2944 stats = stats_proxy_->GetStats();
2945 EXPECT_FALSE(stats.cpu_limited_resolution);
2946 EXPECT_FALSE(stats.cpu_limited_framerate);
2947 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2948
2949 // Force an input frame rate to be available, or the adaptation call won't
2950 // know what framerate to adapt from.
2951 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2952 mock_stats.input_frame_rate = 30;
2953 stats_proxy_->SetMockStats(mock_stats);
2954 video_stream_encoder_->TriggerCpuOveruse();
2955 stats_proxy_->ResetMockStats();
2956 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2957 WaitForEncodedFrame(sequence++);
2958
2959 // We have now adapted once.
2960 stats = stats_proxy_->GetStats();
2961 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2962
2963 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002964 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2965 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002966 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2967 WaitForEncodedFrame(sequence++);
2968 stats = stats_proxy_->GetStats();
2969 EXPECT_FALSE(stats.cpu_limited_resolution);
2970 EXPECT_FALSE(stats.cpu_limited_framerate);
2971 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2972
2973 video_stream_encoder_->Stop();
2974}
2975
2976TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07002977 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02002978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002979 DataRate::BitsPerSec(kTargetBitrateBps),
2980 DataRate::BitsPerSec(kTargetBitrateBps),
2981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002982
asapersson0944a802017-04-07 00:57:58 -07002983 const int kWidth = 1280;
2984 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08002985 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07002986
asaperssonfab67072017-04-04 05:51:49 -07002987 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002988 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07002989 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08002990 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002991 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08002992 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2993
asapersson02465b82017-04-10 01:12:52 -07002994 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002995 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002996 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002997 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08002998 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07002999 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003000 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003001 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3002
3003 // Set new source with adaptation still enabled.
3004 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003006 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003007
3008 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003009 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003010 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003011 stats = stats_proxy_->GetStats();
3012 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003013 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003014 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3015
sprangc5d62e22017-04-02 23:53:04 -07003016 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003018 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003019 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003020 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003021 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003022 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003023 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003024 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003025 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003026 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3027
sprangc5d62e22017-04-02 23:53:04 -07003028 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003029 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003030 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3031 mock_stats.input_frame_rate = 30;
3032 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003033 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003034 stats_proxy_->ResetMockStats();
3035
3036 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003037 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003038 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003039
3040 // Framerate now adapted.
3041 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003042 EXPECT_FALSE(stats.cpu_limited_resolution);
3043 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003044 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3045
3046 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003047 video_stream_encoder_->SetSource(&new_video_source,
3048 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003049 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003050 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003051 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003052
3053 stats = stats_proxy_->GetStats();
3054 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003055 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003056 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3057
3058 // Try to trigger overuse. Should not succeed.
3059 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003060 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003061 stats_proxy_->ResetMockStats();
3062
3063 stats = stats_proxy_->GetStats();
3064 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003065 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003066 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3067
3068 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003069 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003070 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003071 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003072 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003073 stats = stats_proxy_->GetStats();
3074 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003075 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003076 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003077
3078 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003079 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003080 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003081 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003082 stats = stats_proxy_->GetStats();
3083 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003084 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003085 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3086
3087 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003088 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003089 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003090 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003091 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003092 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003093 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003094 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003095 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003096 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003097 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3098
3099 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003100 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003101 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003102 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003103 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003104 stats = stats_proxy_->GetStats();
3105 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003106 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003107 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003108 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003109
mflodmancc3d4422017-08-03 08:27:51 -07003110 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003111}
3112
mflodmancc3d4422017-08-03 08:27:51 -07003113TEST_F(VideoStreamEncoderTest,
3114 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003115 const int kWidth = 1280;
3116 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003118 DataRate::BitsPerSec(kTargetBitrateBps),
3119 DataRate::BitsPerSec(kTargetBitrateBps),
3120 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003121
asaperssonfab67072017-04-04 05:51:49 -07003122 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003123 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003124
asaperssonfab67072017-04-04 05:51:49 -07003125 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003126 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003127
asaperssonfab67072017-04-04 05:51:49 -07003128 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003129 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003130
asaperssonfab67072017-04-04 05:51:49 -07003131 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003132 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003133
kthelgason876222f2016-11-29 01:44:11 -08003134 // Expect a scale down.
3135 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003136 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003137
asapersson02465b82017-04-10 01:12:52 -07003138 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003139 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003140 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003141 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003142
asaperssonfab67072017-04-04 05:51:49 -07003143 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003145 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003146 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003147
asaperssonfab67072017-04-04 05:51:49 -07003148 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003149 EXPECT_EQ(std::numeric_limits<int>::max(),
3150 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003151
asaperssonfab67072017-04-04 05:51:49 -07003152 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003154 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003156
asapersson02465b82017-04-10 01:12:52 -07003157 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003158 EXPECT_EQ(std::numeric_limits<int>::max(),
3159 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003160
mflodmancc3d4422017-08-03 08:27:51 -07003161 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003162}
3163
mflodmancc3d4422017-08-03 08:27:51 -07003164TEST_F(VideoStreamEncoderTest,
3165 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003166 const int kWidth = 1280;
3167 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003168 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003169 DataRate::BitsPerSec(kTargetBitrateBps),
3170 DataRate::BitsPerSec(kTargetBitrateBps),
3171 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003172
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003173 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003174 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003175 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003176 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003177
3178 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003179 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003180 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003181 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3182 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3183
3184 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003185 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003186 EXPECT_THAT(source.sink_wants(),
3187 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003188 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3189 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3190 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3191
3192 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003194 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3195 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3196 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3197
mflodmancc3d4422017-08-03 08:27:51 -07003198 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003199}
3200
mflodmancc3d4422017-08-03 08:27:51 -07003201TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003202 const int kWidth = 1280;
3203 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003205 DataRate::BitsPerSec(kTargetBitrateBps),
3206 DataRate::BitsPerSec(kTargetBitrateBps),
3207 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003208
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003209 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003210 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003211 video_stream_encoder_->SetSource(&source,
3212 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003213 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3214 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003215 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003216
3217 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003218 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003219 EXPECT_THAT(source.sink_wants(),
3220 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003221 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3222 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3223 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3224
3225 // Trigger adapt down for same input resolution, expect no change.
3226 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3227 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003228 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003229 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3230 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3231 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3232
3233 // Trigger adapt down for larger input resolution, expect no change.
3234 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3235 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003236 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003237 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3238 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3239 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3240
mflodmancc3d4422017-08-03 08:27:51 -07003241 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003242}
3243
mflodmancc3d4422017-08-03 08:27:51 -07003244TEST_F(VideoStreamEncoderTest,
3245 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003246 const int kWidth = 1280;
3247 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003248 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003249 DataRate::BitsPerSec(kTargetBitrateBps),
3250 DataRate::BitsPerSec(kTargetBitrateBps),
3251 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003252
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003253 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003254 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003256 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003257
3258 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003259 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003260 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003261 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3262 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3263
3264 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003265 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003266 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003267 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3268 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3269
mflodmancc3d4422017-08-03 08:27:51 -07003270 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003271}
3272
mflodmancc3d4422017-08-03 08:27:51 -07003273TEST_F(VideoStreamEncoderTest,
3274 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003275 const int kWidth = 1280;
3276 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003278 DataRate::BitsPerSec(kTargetBitrateBps),
3279 DataRate::BitsPerSec(kTargetBitrateBps),
3280 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003281
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003282 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003283 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003284 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003285 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003286
3287 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003288 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003289 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003290 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003291 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3292
3293 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003294 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003295 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003296 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003297 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003300}
3301
mflodmancc3d4422017-08-03 08:27:51 -07003302TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003303 const int kWidth = 1280;
3304 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003306 DataRate::BitsPerSec(kTargetBitrateBps),
3307 DataRate::BitsPerSec(kTargetBitrateBps),
3308 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003309
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003310 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003311 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003312 video_stream_encoder_->SetSource(&source,
3313 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003314
3315 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3316 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003318 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3319 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3320 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3321
3322 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003324 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003325 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3326 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3327 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3328
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003330}
3331
mflodmancc3d4422017-08-03 08:27:51 -07003332TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003333 const int kWidth = 1280;
3334 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003336 DataRate::BitsPerSec(kTargetBitrateBps),
3337 DataRate::BitsPerSec(kTargetBitrateBps),
3338 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003339
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003340 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003341 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003342 video_stream_encoder_->SetSource(&source,
3343 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003344
3345 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3346 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003347 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3349 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3350 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
3352 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003353 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003354 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3356 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3357 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3358
mflodmancc3d4422017-08-03 08:27:51 -07003359 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003360}
3361
mflodmancc3d4422017-08-03 08:27:51 -07003362TEST_F(VideoStreamEncoderTest,
3363 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003364 const int kWidth = 1280;
3365 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003366 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003367 DataRate::BitsPerSec(kTargetBitrateBps),
3368 DataRate::BitsPerSec(kTargetBitrateBps),
3369 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003370
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003371 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003372 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003373 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003374 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003375 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003376
3377 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003378 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003379 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003385 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003386 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003387 EXPECT_THAT(source.sink_wants(),
3388 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3391
3392 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003393 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003394 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003395 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3396 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3397 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3398
mflodmancc3d4422017-08-03 08:27:51 -07003399 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003400}
3401
mflodmancc3d4422017-08-03 08:27:51 -07003402TEST_F(VideoStreamEncoderTest,
3403 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003404 const int kWidth = 1280;
3405 const int kHeight = 720;
3406 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003408 DataRate::BitsPerSec(kTargetBitrateBps),
3409 DataRate::BitsPerSec(kTargetBitrateBps),
3410 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003411
3412 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3413 stats.input_frame_rate = kInputFps;
3414 stats_proxy_->SetMockStats(stats);
3415
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003416 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003417 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3418 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003419 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003420
3421 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003422 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003423 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3424 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003425 EXPECT_THAT(video_source_.sink_wants(),
3426 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003427
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003428 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003429 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003430 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003431 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003432 // Give the encoder queue time to process the change in degradation preference
3433 // by waiting for an encoded frame.
3434 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3435 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003436 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003437
3438 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003439 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003440 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3441 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003442 EXPECT_THAT(new_video_source.sink_wants(),
3443 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003444
3445 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003446 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003447 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003448
mflodmancc3d4422017-08-03 08:27:51 -07003449 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003450}
3451
mflodmancc3d4422017-08-03 08:27:51 -07003452TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003453 const int kWidth = 1280;
3454 const int kHeight = 720;
3455 const size_t kNumFrames = 10;
3456
Henrik Boström381d1092020-05-12 18:49:07 +02003457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003458 DataRate::BitsPerSec(kTargetBitrateBps),
3459 DataRate::BitsPerSec(kTargetBitrateBps),
3460 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003461
asaperssond0de2952017-04-21 01:47:31 -07003462 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003463 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003464 video_source_.set_adaptation_enabled(true);
3465
3466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3467 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 int downscales = 0;
3470 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003471 video_source_.IncomingCapturedFrame(
3472 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3473 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003474
asaperssonfab67072017-04-04 05:51:49 -07003475 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003476 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003477 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003478 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003479
3480 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3481 ++downscales;
3482
3483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3484 EXPECT_EQ(downscales,
3485 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3486 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003487 }
mflodmancc3d4422017-08-03 08:27:51 -07003488 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003489}
3490
mflodmancc3d4422017-08-03 08:27:51 -07003491TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003492 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3493 const int kWidth = 1280;
3494 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003496 DataRate::BitsPerSec(kTargetBitrateBps),
3497 DataRate::BitsPerSec(kTargetBitrateBps),
3498 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003499
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003500 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003501 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003502 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003503 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003504 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003505
Åsa Persson8c1bf952018-09-13 10:42:19 +02003506 int64_t timestamp_ms = kFrameIntervalMs;
3507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003508 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003509 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003510 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3511 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3512
3513 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003514 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003515 timestamp_ms += kFrameIntervalMs;
3516 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3517 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003518 EXPECT_THAT(source.sink_wants(),
3519 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003520 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3521 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3522
3523 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003524 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003525 timestamp_ms += kFrameIntervalMs;
3526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003527 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003529 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3530 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3531
3532 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003533 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003534 timestamp_ms += kFrameIntervalMs;
3535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3536 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003537 EXPECT_THAT(source.sink_wants(),
3538 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003539 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3540 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3541
3542 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003543 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003544 timestamp_ms += kFrameIntervalMs;
3545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003546 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003548 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3549 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3550
mflodmancc3d4422017-08-03 08:27:51 -07003551 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003552}
3553
mflodmancc3d4422017-08-03 08:27:51 -07003554TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003555 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3556 const int kWidth = 1280;
3557 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003559 DataRate::BitsPerSec(kTargetBitrateBps),
3560 DataRate::BitsPerSec(kTargetBitrateBps),
3561 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003562
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003563 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003564 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003565 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003566 video_stream_encoder_->SetSource(&source,
3567 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003568
Åsa Persson8c1bf952018-09-13 10:42:19 +02003569 int64_t timestamp_ms = kFrameIntervalMs;
3570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003571 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003572 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3574 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3575
3576 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003577 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003581 EXPECT_THAT(source.sink_wants(),
3582 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3584 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585
3586 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003587 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003588 timestamp_ms += kFrameIntervalMs;
3589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003590 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003591 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003592 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3593 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3594
3595 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003596 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003597 timestamp_ms += kFrameIntervalMs;
3598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3599 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003600 EXPECT_THAT(source.sink_wants(),
3601 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3603 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3604
3605 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003606 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003607 timestamp_ms += kFrameIntervalMs;
3608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003609 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003610 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3612 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613
mflodmancc3d4422017-08-03 08:27:51 -07003614 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003615}
3616
Sergey Silkin41c650b2019-10-14 13:12:19 +02003617TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3618 fake_encoder_.SetResolutionBitrateLimits(
3619 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3620
Henrik Boström381d1092020-05-12 18:49:07 +02003621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003622 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3623 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3624 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3625 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003626
3627 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003628 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003629 source.set_adaptation_enabled(true);
3630 video_stream_encoder_->SetSource(
3631 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3632
3633 // Insert 720p frame.
3634 int64_t timestamp_ms = kFrameIntervalMs;
3635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3636 WaitForEncodedFrame(1280, 720);
3637
3638 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003640 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3641 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3642 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3643 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003644 video_stream_encoder_->TriggerQualityLow();
3645
3646 // Insert 720p frame. It should be downscaled and encoded.
3647 timestamp_ms += kFrameIntervalMs;
3648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3649 WaitForEncodedFrame(960, 540);
3650
3651 // Trigger adapt up. Higher resolution should not be requested duo to lack
3652 // of bitrate.
3653 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003654 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003655
3656 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003658 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3659 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3660 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3661 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003662
3663 // Trigger adapt up. Higher resolution should be requested.
3664 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003665 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003666
3667 video_stream_encoder_->Stop();
3668}
3669
3670TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3671 fake_encoder_.SetResolutionBitrateLimits(
3672 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3673
3674 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003676 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3677 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3678 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3679 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003680
3681 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003682 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003683 source.set_adaptation_enabled(true);
3684 video_stream_encoder_->SetSource(
3685 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3686
3687 // Insert 720p frame. It should be dropped and lower resolution should be
3688 // requested.
3689 int64_t timestamp_ms = kFrameIntervalMs;
3690 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3691 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003692 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003693
3694 // Insert 720p frame. It should be downscaled and encoded.
3695 timestamp_ms += kFrameIntervalMs;
3696 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3697 WaitForEncodedFrame(960, 540);
3698
3699 video_stream_encoder_->Stop();
3700}
3701
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003702class BalancedDegradationTest : public VideoStreamEncoderTest {
3703 protected:
3704 void SetupTest() {
3705 // Reset encoder for field trials to take effect.
3706 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003707 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003708
3709 // Enable BALANCED preference.
3710 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003711 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3712 }
3713
3714 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003716 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3717 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003718 }
3719
Åsa Persson45b176f2019-09-30 11:19:05 +02003720 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003721 timestamp_ms_ += kFrameIntervalMs;
3722 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003723 }
3724
3725 void InsertFrameAndWaitForEncoded() {
3726 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003727 sink_.WaitForEncodedFrame(timestamp_ms_);
3728 }
3729
3730 const int kWidth = 640; // pixels:640x360=230400
3731 const int kHeight = 360;
3732 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3733 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003734 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003735};
3736
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003737TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003738 test::ScopedFieldTrials field_trials(
3739 "WebRTC-Video-BalancedDegradationSettings/"
3740 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3741 SetupTest();
3742
3743 // Force input frame rate.
3744 const int kInputFps = 24;
3745 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3746 stats.input_frame_rate = kInputFps;
3747 stats_proxy_->SetMockStats(stats);
3748
Åsa Persson45b176f2019-09-30 11:19:05 +02003749 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003750 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003751
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003752 // Trigger adapt down, expect scaled down framerate and resolution,
3753 // since Fps diff (input-requested:0) < threshold.
3754 video_stream_encoder_->TriggerQualityLow();
3755 EXPECT_THAT(source_.sink_wants(),
3756 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003757
3758 video_stream_encoder_->Stop();
3759}
3760
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003761TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003762 test::ScopedFieldTrials field_trials(
3763 "WebRTC-Video-BalancedDegradationSettings/"
3764 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3765 SetupTest();
3766
3767 // Force input frame rate.
3768 const int kInputFps = 25;
3769 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3770 stats.input_frame_rate = kInputFps;
3771 stats_proxy_->SetMockStats(stats);
3772
Åsa Persson45b176f2019-09-30 11:19:05 +02003773 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003774 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003775
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003776 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3777 // Fps diff (input-requested:1) == threshold.
3778 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003779 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003780
3781 video_stream_encoder_->Stop();
3782}
3783
3784TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3785 test::ScopedFieldTrials field_trials(
3786 "WebRTC-Video-BalancedDegradationSettings/"
3787 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3788 SetupTest();
3789
3790 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3791
Åsa Persson45b176f2019-09-30 11:19:05 +02003792 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003793 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003794
3795 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3796 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003797 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003798
3799 video_stream_encoder_->Stop();
3800}
3801
Åsa Perssonccfb3402019-09-25 15:13:04 +02003802TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003803 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003804 "WebRTC-Video-BalancedDegradationSettings/"
3805 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003806 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003807
Åsa Persson1b247f12019-08-14 17:26:39 +02003808 const int kMinBitrateBps = 425000;
3809 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003810 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003811
Åsa Persson45b176f2019-09-30 11:19:05 +02003812 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003813 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3815
3816 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3817 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003818 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003819 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003820 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3821
3822 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3823 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003824 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003825 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003826 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3827
Åsa Persson30ab0152019-08-27 12:22:33 +02003828 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3829 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003830 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003831 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003832 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003833 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3834
3835 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003836 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003837 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003838 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003839
Åsa Persson30ab0152019-08-27 12:22:33 +02003840 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003841 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003842 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003843 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003844 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003845 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3846
3847 video_stream_encoder_->Stop();
3848}
3849
Åsa Perssonccfb3402019-09-25 15:13:04 +02003850TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003851 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3852 test::ScopedFieldTrials field_trials(
3853 "WebRTC-Video-BalancedDegradationSettings/"
3854 "pixels:57600|129600|230400,fps:7|24|24/");
3855 SetupTest();
3856 OnBitrateUpdated(kLowTargetBitrateBps);
3857
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003858 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003859
3860 // Insert frame, expect scaled down:
3861 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3862 InsertFrame();
3863 EXPECT_FALSE(WaitForFrame(1000));
3864 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3865 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3866
3867 // Insert frame, expect scaled down:
3868 // resolution (320x180@24fps).
3869 InsertFrame();
3870 EXPECT_FALSE(WaitForFrame(1000));
3871 EXPECT_LT(source_.sink_wants().max_pixel_count,
3872 source_.last_wants().max_pixel_count);
3873 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3874
3875 // Frame should not be dropped (min pixels per frame reached).
3876 InsertFrameAndWaitForEncoded();
3877
3878 video_stream_encoder_->Stop();
3879}
3880
3881TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003882 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003883 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003884 "WebRTC-Video-BalancedDegradationSettings/"
3885 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003886 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003887
Åsa Persson30ab0152019-08-27 12:22:33 +02003888 const int kResolutionMinBitrateBps = 435000;
3889 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003890 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003891
Åsa Persson45b176f2019-09-30 11:19:05 +02003892 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003893 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003894 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3895
3896 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3897 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003898 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003900 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3901
3902 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3903 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003904 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003905 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003906 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3907
3908 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3909 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003910 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003911 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003912 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913
Åsa Persson30ab0152019-08-27 12:22:33 +02003914 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
3915 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003916 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003917 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003918 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3919
3920 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
3921 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003922 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3924
3925 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003926 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003927 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003928 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003929 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003930 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3931
3932 video_stream_encoder_->Stop();
3933}
3934
Åsa Perssonccfb3402019-09-25 15:13:04 +02003935TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003936 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003937 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003938 "WebRTC-Video-BalancedDegradationSettings/"
3939 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003940 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003941
Åsa Persson30ab0152019-08-27 12:22:33 +02003942 const int kMinBitrateBps = 425000;
3943 const int kTooLowMinBitrateBps = 424000;
3944 const int kResolutionMinBitrateBps = 435000;
3945 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003946 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003947
Åsa Persson45b176f2019-09-30 11:19:05 +02003948 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003949 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003950 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3951
3952 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3953 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003954 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003955 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003956 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3957
3958 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3959 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003960 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003961 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003962 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3963
3964 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3965 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003966 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003967 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003968 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3969
3970 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
3971 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003972 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003973 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3974
3975 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003976 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003977 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003978 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003979 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003980 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3981
3982 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003983 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003984 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003985 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003986 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3987
3988 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003989 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003990 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003991 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003992 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02003993 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3994
Åsa Persson1b247f12019-08-14 17:26:39 +02003995 video_stream_encoder_->Stop();
3996}
3997
mflodmancc3d4422017-08-03 08:27:51 -07003998TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003999 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4000 const int kWidth = 1280;
4001 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004003 DataRate::BitsPerSec(kTargetBitrateBps),
4004 DataRate::BitsPerSec(kTargetBitrateBps),
4005 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004006
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004007 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004008 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004009 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004010 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004011 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004012
Åsa Persson8c1bf952018-09-13 10:42:19 +02004013 int64_t timestamp_ms = kFrameIntervalMs;
4014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004015 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004016 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004017 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4018 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4019 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4020 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4021
4022 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004023 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004024 timestamp_ms += kFrameIntervalMs;
4025 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4026 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004027 EXPECT_THAT(source.sink_wants(),
4028 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004029 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4030 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4031 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4032 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4033
4034 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004036 timestamp_ms += kFrameIntervalMs;
4037 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4038 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004039 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004040 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4041 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4042 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4044
Jonathan Yubc771b72017-12-08 17:04:29 -08004045 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004047 timestamp_ms += kFrameIntervalMs;
4048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4049 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004051 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4052 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004053 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004054 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4055
Jonathan Yubc771b72017-12-08 17:04:29 -08004056 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004057 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004058 timestamp_ms += kFrameIntervalMs;
4059 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4060 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004061 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004062 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004063 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4064 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4065 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4066 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4067
Jonathan Yubc771b72017-12-08 17:04:29 -08004068 // Trigger quality adapt down, expect no change (min resolution reached).
4069 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004070 timestamp_ms += kFrameIntervalMs;
4071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4072 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004073 EXPECT_THAT(source.sink_wants(), FpsMax());
4074 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004075 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4077 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4078 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4079
Evan Shrubsole64469032020-06-11 10:45:29 +02004080 // Trigger quality adapt up, expect upscaled resolution (480x270).
4081 video_stream_encoder_->TriggerQualityHigh();
4082 timestamp_ms += kFrameIntervalMs;
4083 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4084 WaitForEncodedFrame(timestamp_ms);
4085 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4086 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4088 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4089 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4090
4091 // Trigger quality and cpu adapt up since both are most limited, expect
4092 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004093 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004094 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004095 timestamp_ms += kFrameIntervalMs;
4096 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4097 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004098 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004099 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4100 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4101 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004102 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004103
Evan Shrubsole64469032020-06-11 10:45:29 +02004104 // Trigger quality and cpu adapt up since both are most limited, expect
4105 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004106 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004107 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004108 timestamp_ms += kFrameIntervalMs;
4109 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4110 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004111 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004112 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004113 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004114 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004115 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4116 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004117
Evan Shrubsole64469032020-06-11 10:45:29 +02004118 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4119 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004120 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004121 timestamp_ms += kFrameIntervalMs;
4122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4123 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004124 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004125 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4126 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004127 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004128 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004129
4130 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004131 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004132 timestamp_ms += kFrameIntervalMs;
4133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004135 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004136 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004137 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4138 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004139 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004140 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004141
mflodmancc3d4422017-08-03 08:27:51 -07004142 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004143}
4144
mflodmancc3d4422017-08-03 08:27:51 -07004145TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004146 const int kWidth = 640;
4147 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004148
Henrik Boström381d1092020-05-12 18:49:07 +02004149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004150 DataRate::BitsPerSec(kTargetBitrateBps),
4151 DataRate::BitsPerSec(kTargetBitrateBps),
4152 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004153
perkj803d97f2016-11-01 11:45:46 -07004154 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004155 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004156 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004157 }
4158
mflodmancc3d4422017-08-03 08:27:51 -07004159 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004160 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004161 video_source_.IncomingCapturedFrame(CreateFrame(
4162 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004163 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004164 }
4165
mflodmancc3d4422017-08-03 08:27:51 -07004166 video_stream_encoder_->Stop();
4167 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004168 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004169
Ying Wangef3998f2019-12-09 13:06:53 +01004170 EXPECT_METRIC_EQ(
4171 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4172 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004173 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4174}
4175
mflodmancc3d4422017-08-03 08:27:51 -07004176TEST_F(VideoStreamEncoderTest,
4177 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004179 DataRate::BitsPerSec(kTargetBitrateBps),
4180 DataRate::BitsPerSec(kTargetBitrateBps),
4181 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004182 const int kWidth = 640;
4183 const int kHeight = 360;
4184
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004185 video_stream_encoder_->SetSource(&video_source_,
4186 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004187
4188 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4189 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004190 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004191 }
4192
mflodmancc3d4422017-08-03 08:27:51 -07004193 video_stream_encoder_->Stop();
4194 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004195 stats_proxy_.reset();
4196
4197 EXPECT_EQ(0,
4198 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4199}
4200
Per Kjellanderdcef6412020-10-07 15:09:05 +02004201TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4202 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004203 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004204 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004205
4206 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004207 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004208 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004209 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4210 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004211
Henrik Boström381d1092020-05-12 18:49:07 +02004212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004213 DataRate::BitsPerSec(kLowTargetBitrateBps),
4214 DataRate::BitsPerSec(kLowTargetBitrateBps),
4215 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004216
sprang57c2fff2017-01-16 06:24:02 -08004217 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004218 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4219 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004220 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4221 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4222
Erik Språngd7329ca2019-02-21 21:19:53 +01004223 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004224 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004225 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004226
Per Kjellanderdcef6412020-10-07 15:09:05 +02004227 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004228 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004229 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4230 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004231 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004232 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004233
Per Kjellanderdcef6412020-10-07 15:09:05 +02004234 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004235 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004236 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004237 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004238 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4239 WaitForEncodedFrame(CurrentTimeMs());
4240 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004241 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004242 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004243
mflodmancc3d4422017-08-03 08:27:51 -07004244 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004245}
4246
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004247TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004248 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004249 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004250 kVideoLayersAllocation);
4251
4252 const int kDefaultFps = 30;
4253
4254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4255 DataRate::BitsPerSec(kLowTargetBitrateBps),
4256 DataRate::BitsPerSec(kLowTargetBitrateBps),
4257 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4258
4259 video_source_.IncomingCapturedFrame(
4260 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4261 WaitForEncodedFrame(CurrentTimeMs());
4262 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4263 VideoLayersAllocation last_layer_allocation =
4264 sink_.GetLastVideoLayersAllocation();
4265 // kLowTargetBitrateBps is only enough for one spatial layer.
4266 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4267
4268 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004269 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004270 // Check that encoder has been updated too, not just allocation observer.
4271 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4272 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4273
Erik Språng9d69cbe2020-10-22 17:44:42 +02004274 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004275 int number_of_layers_allocation = 1;
4276 const int64_t start_time_ms = CurrentTimeMs();
4277 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4278 video_source_.IncomingCapturedFrame(
4279 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4280 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004281 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4282 number_of_layers_allocation = sink_.number_of_layers_allocations();
4283 VideoLayersAllocation new_allocation =
4284 sink_.GetLastVideoLayersAllocation();
4285 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4286 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4287 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4288 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4289 .target_bitrate_per_temporal_layer,
4290 last_layer_allocation.active_spatial_layers[0]
4291 .target_bitrate_per_temporal_layer);
4292 last_layer_allocation = new_allocation;
4293 }
4294 }
4295 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4296 video_stream_encoder_->Stop();
4297}
4298
4299TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004300 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004301 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4302 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4303 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004304 VideoEncoderConfig video_encoder_config;
4305 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4306 /* num_streams*/ 3, &video_encoder_config);
4307 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4308 video_encoder_config.content_type =
4309 VideoEncoderConfig::ContentType::kRealtimeVideo;
4310 video_encoder_config.encoder_specific_settings =
4311 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4312 VideoEncoder::GetDefaultVp8Settings());
4313 for (auto& layer : video_encoder_config.simulcast_layers) {
4314 layer.num_temporal_layers = 2;
4315 }
4316 // Simulcast layers are used for enabling/disabling streams.
4317 video_encoder_config.simulcast_layers[0].active = true;
4318 video_encoder_config.simulcast_layers[1].active = false;
4319 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004320 ConfigureEncoder(std::move(video_encoder_config),
4321 VideoStreamEncoder::BitrateAllocationCallbackType::
4322 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004323
4324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4325 DataRate::BitsPerSec(kTargetBitrateBps),
4326 DataRate::BitsPerSec(kTargetBitrateBps),
4327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4328
4329 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4330 WaitForEncodedFrame(CurrentTimeMs());
4331 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4332 VideoLayersAllocation last_layer_allocation =
4333 sink_.GetLastVideoLayersAllocation();
4334
4335 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4336 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4337 .target_bitrate_per_temporal_layer,
4338 SizeIs(2));
4339 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4340 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4341 video_stream_encoder_->Stop();
4342}
4343
4344TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004345 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004346 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4347 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4348 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004349 VideoEncoderConfig video_encoder_config;
4350 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4351 /* num_streams*/ 3, &video_encoder_config);
4352 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4353 video_encoder_config.content_type =
4354 VideoEncoderConfig::ContentType::kRealtimeVideo;
4355 video_encoder_config.encoder_specific_settings =
4356 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4357 VideoEncoder::GetDefaultVp8Settings());
4358 for (auto& layer : video_encoder_config.simulcast_layers) {
4359 layer.num_temporal_layers = 2;
4360 }
4361 // Simulcast layers are used for enabling/disabling streams.
4362 video_encoder_config.simulcast_layers[0].active = true;
4363 video_encoder_config.simulcast_layers[1].active = false;
4364 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004365 ConfigureEncoder(std::move(video_encoder_config),
4366 VideoStreamEncoder::BitrateAllocationCallbackType::
4367 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004368
4369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4370 DataRate::BitsPerSec(kTargetBitrateBps),
4371 DataRate::BitsPerSec(kTargetBitrateBps),
4372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4373
4374 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4375 WaitForEncodedFrame(CurrentTimeMs());
4376 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4377 VideoLayersAllocation last_layer_allocation =
4378 sink_.GetLastVideoLayersAllocation();
4379
4380 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4381 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4382 .target_bitrate_per_temporal_layer,
4383 SizeIs(2));
4384 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4385
4386 video_stream_encoder_->Stop();
4387}
4388
4389TEST_F(VideoStreamEncoderTest,
4390 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4391 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4392 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004393 VideoEncoderConfig video_encoder_config;
4394 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4395 /* num_streams*/ 1, &video_encoder_config);
4396 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4397 video_encoder_config.content_type =
4398 VideoEncoderConfig::ContentType::kRealtimeVideo;
4399 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4400 vp9_settings.numberOfSpatialLayers = 2;
4401 vp9_settings.numberOfTemporalLayers = 2;
4402 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4403 vp9_settings.automaticResizeOn = false;
4404 video_encoder_config.encoder_specific_settings =
4405 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4406 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004407 ConfigureEncoder(std::move(video_encoder_config),
4408 VideoStreamEncoder::BitrateAllocationCallbackType::
4409 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004410
4411 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4412 DataRate::BitsPerSec(kTargetBitrateBps),
4413 DataRate::BitsPerSec(kTargetBitrateBps),
4414 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4415
4416 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4417 WaitForEncodedFrame(CurrentTimeMs());
4418 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4419 VideoLayersAllocation last_layer_allocation =
4420 sink_.GetLastVideoLayersAllocation();
4421
4422 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4423 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4424 .target_bitrate_per_temporal_layer,
4425 SizeIs(2));
4426 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4427 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4428 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4429 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4430 .target_bitrate_per_temporal_layer,
4431 SizeIs(2));
4432 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4433 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4434 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4435
4436 // Since full SVC is used, expect the top layer to utilize the full target
4437 // rate.
4438 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4439 .target_bitrate_per_temporal_layer[1],
4440 DataRate::BitsPerSec(kTargetBitrateBps));
4441 video_stream_encoder_->Stop();
4442}
4443
4444TEST_F(VideoStreamEncoderTest,
4445 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4446 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4447 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004448 VideoEncoderConfig video_encoder_config;
4449 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4450 /* num_streams*/ 1, &video_encoder_config);
4451 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4452 video_encoder_config.content_type =
4453 VideoEncoderConfig::ContentType::kRealtimeVideo;
4454 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4455 vp9_settings.numberOfSpatialLayers = 2;
4456 vp9_settings.numberOfTemporalLayers = 2;
4457 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4458 vp9_settings.automaticResizeOn = false;
4459 video_encoder_config.encoder_specific_settings =
4460 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4461 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004462 ConfigureEncoder(std::move(video_encoder_config),
4463 VideoStreamEncoder::BitrateAllocationCallbackType::
4464 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004465
4466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4467 DataRate::BitsPerSec(kTargetBitrateBps),
4468 DataRate::BitsPerSec(kTargetBitrateBps),
4469 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4470
4471 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4472 WaitForEncodedFrame(CurrentTimeMs());
4473 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4474 VideoLayersAllocation last_layer_allocation =
4475 sink_.GetLastVideoLayersAllocation();
4476
4477 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4478 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4479 .target_bitrate_per_temporal_layer,
4480 SizeIs(1));
4481 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4482 .target_bitrate_per_temporal_layer,
4483 SizeIs(1));
4484 // Since full SVC is used, expect the top layer to utilize the full target
4485 // rate.
4486 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4487 .target_bitrate_per_temporal_layer[0],
4488 DataRate::BitsPerSec(kTargetBitrateBps));
4489 video_stream_encoder_->Stop();
4490}
4491
4492TEST_F(VideoStreamEncoderTest,
4493 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4494 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4495 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004496 VideoEncoderConfig video_encoder_config;
4497 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4498 /* num_streams*/ 1, &video_encoder_config);
4499 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4500 video_encoder_config.content_type =
4501 VideoEncoderConfig::ContentType::kRealtimeVideo;
4502 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4503 vp9_settings.numberOfSpatialLayers = 2;
4504 vp9_settings.numberOfTemporalLayers = 2;
4505 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4506 vp9_settings.automaticResizeOn = false;
4507 video_encoder_config.encoder_specific_settings =
4508 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4509 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004510 ConfigureEncoder(std::move(video_encoder_config),
4511 VideoStreamEncoder::BitrateAllocationCallbackType::
4512 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004513
4514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4515 DataRate::BitsPerSec(kTargetBitrateBps),
4516 DataRate::BitsPerSec(kTargetBitrateBps),
4517 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4518
4519 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4520 WaitForEncodedFrame(CurrentTimeMs());
4521 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4522 VideoLayersAllocation last_layer_allocation =
4523 sink_.GetLastVideoLayersAllocation();
4524
4525 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4526 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4527 .target_bitrate_per_temporal_layer,
4528 SizeIs(2));
4529 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4530 .target_bitrate_per_temporal_layer,
4531 SizeIs(2));
4532 // Since KSVC is, spatial layers are independend except on key frames.
4533 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4534 .target_bitrate_per_temporal_layer[1],
4535 DataRate::BitsPerSec(kTargetBitrateBps));
4536 video_stream_encoder_->Stop();
4537}
4538
4539TEST_F(VideoStreamEncoderTest,
4540 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4541 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4542 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4543 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004544 VideoEncoderConfig video_encoder_config;
4545 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4546 /* num_streams*/ 1, &video_encoder_config);
4547 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4548 video_encoder_config.content_type =
4549 VideoEncoderConfig::ContentType::kRealtimeVideo;
4550 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4551 vp9_settings.numberOfSpatialLayers = 3;
4552 vp9_settings.numberOfTemporalLayers = 2;
4553 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4554 vp9_settings.automaticResizeOn = false;
4555 video_encoder_config.encoder_specific_settings =
4556 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4557 vp9_settings);
4558 // Simulcast layers are used for enabling/disabling streams.
4559 video_encoder_config.simulcast_layers.resize(3);
4560 video_encoder_config.simulcast_layers[0].active = false;
4561 video_encoder_config.simulcast_layers[1].active = true;
4562 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004563 ConfigureEncoder(std::move(video_encoder_config),
4564 VideoStreamEncoder::BitrateAllocationCallbackType::
4565 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004566
4567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4568 DataRate::BitsPerSec(kTargetBitrateBps),
4569 DataRate::BitsPerSec(kTargetBitrateBps),
4570 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4571
4572 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4573 WaitForEncodedFrame(CurrentTimeMs());
4574 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4575 VideoLayersAllocation last_layer_allocation =
4576 sink_.GetLastVideoLayersAllocation();
4577
4578 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4579 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4580 .target_bitrate_per_temporal_layer,
4581 SizeIs(2));
4582 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4583 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4584
4585 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4586 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4587 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4588 .target_bitrate_per_temporal_layer,
4589 SizeIs(2));
4590 // Since full SVC is used, expect the top layer to utilize the full target
4591 // rate.
4592 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4593 .target_bitrate_per_temporal_layer[1],
4594 DataRate::BitsPerSec(kTargetBitrateBps));
4595 video_stream_encoder_->Stop();
4596}
4597
4598TEST_F(VideoStreamEncoderTest,
4599 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4600 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4601 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4602 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004603 VideoEncoderConfig video_encoder_config;
4604 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4605 /* num_streams*/ 1, &video_encoder_config);
4606 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4607 video_encoder_config.content_type =
4608 VideoEncoderConfig::ContentType::kRealtimeVideo;
4609 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4610 vp9_settings.numberOfSpatialLayers = 3;
4611 vp9_settings.numberOfTemporalLayers = 2;
4612 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4613 vp9_settings.automaticResizeOn = false;
4614 video_encoder_config.encoder_specific_settings =
4615 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4616 vp9_settings);
4617 // Simulcast layers are used for enabling/disabling streams.
4618 video_encoder_config.simulcast_layers.resize(3);
4619 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004620 ConfigureEncoder(std::move(video_encoder_config),
4621 VideoStreamEncoder::BitrateAllocationCallbackType::
4622 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004623
4624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4625 DataRate::BitsPerSec(kTargetBitrateBps),
4626 DataRate::BitsPerSec(kTargetBitrateBps),
4627 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4628
4629 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4630 WaitForEncodedFrame(CurrentTimeMs());
4631 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4632 VideoLayersAllocation last_layer_allocation =
4633 sink_.GetLastVideoLayersAllocation();
4634
4635 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4636 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4637 .target_bitrate_per_temporal_layer,
4638 SizeIs(2));
4639 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4640 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4641
4642 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4643 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4644 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4645 .target_bitrate_per_temporal_layer,
4646 SizeIs(2));
4647 video_stream_encoder_->Stop();
4648}
4649
4650TEST_F(VideoStreamEncoderTest,
4651 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4652 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4653 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4654 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004655 VideoEncoderConfig video_encoder_config;
4656 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4657 /* num_streams*/ 1, &video_encoder_config);
4658 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4659 video_encoder_config.content_type =
4660 VideoEncoderConfig::ContentType::kRealtimeVideo;
4661 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4662 vp9_settings.numberOfSpatialLayers = 3;
4663 vp9_settings.numberOfTemporalLayers = 2;
4664 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4665 vp9_settings.automaticResizeOn = false;
4666 video_encoder_config.encoder_specific_settings =
4667 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4668 vp9_settings);
4669 // Simulcast layers are used for enabling/disabling streams.
4670 video_encoder_config.simulcast_layers.resize(3);
4671 video_encoder_config.simulcast_layers[0].active = false;
4672 video_encoder_config.simulcast_layers[1].active = false;
4673 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004674 ConfigureEncoder(std::move(video_encoder_config),
4675 VideoStreamEncoder::BitrateAllocationCallbackType::
4676 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004677
4678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4679 DataRate::BitsPerSec(kTargetBitrateBps),
4680 DataRate::BitsPerSec(kTargetBitrateBps),
4681 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4682
4683 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4684 WaitForEncodedFrame(CurrentTimeMs());
4685 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4686 VideoLayersAllocation last_layer_allocation =
4687 sink_.GetLastVideoLayersAllocation();
4688
4689 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4690 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4691 .target_bitrate_per_temporal_layer,
4692 SizeIs(2));
4693 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4694 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4695 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4696 .target_bitrate_per_temporal_layer[1],
4697 DataRate::BitsPerSec(kTargetBitrateBps));
4698 video_stream_encoder_->Stop();
4699}
4700
4701TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4702 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004703 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004704 kVideoLayersAllocation);
4705 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4706 DataRate::BitsPerSec(kTargetBitrateBps),
4707 DataRate::BitsPerSec(kTargetBitrateBps),
4708 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4709
4710 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4711 WaitForEncodedFrame(CurrentTimeMs());
4712 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4713 VideoLayersAllocation last_layer_allocation =
4714 sink_.GetLastVideoLayersAllocation();
4715
4716 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4717 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4718 .target_bitrate_per_temporal_layer,
4719 SizeIs(1));
4720 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4721 .target_bitrate_per_temporal_layer[0],
4722 DataRate::BitsPerSec(kTargetBitrateBps));
4723 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4724 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4725 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4726 video_stream_encoder_->Stop();
4727}
4728
4729TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004730 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4731 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004732 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004733 kVideoLayersAllocation);
4734
4735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4736 DataRate::BitsPerSec(kLowTargetBitrateBps),
4737 DataRate::BitsPerSec(kLowTargetBitrateBps),
4738 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4739
4740 video_source_.IncomingCapturedFrame(
4741 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4742 WaitForEncodedFrame(CurrentTimeMs());
4743 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4744 VideoLayersAllocation last_layer_allocation =
4745 sink_.GetLastVideoLayersAllocation();
4746 // kLowTargetBitrateBps is only enough for one spatial layer.
4747 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4748 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4749 .target_bitrate_per_temporal_layer[0],
4750 DataRate::BitsPerSec(kLowTargetBitrateBps));
4751
4752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4753 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4754 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4755 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4756 video_source_.IncomingCapturedFrame(
4757 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4758 WaitForEncodedFrame(CurrentTimeMs());
4759
4760 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4761 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4762 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4763 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4764 .target_bitrate_per_temporal_layer[0],
4765 DataRate::Zero());
4766
4767 video_stream_encoder_->Stop();
4768}
4769
Per Kjellander4190ce92020-12-15 17:24:55 +01004770TEST_F(VideoStreamEncoderTest,
4771 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4772 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004773 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004774 kVideoLayersAllocation);
4775
4776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4777 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4778 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4779 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4780
4781 video_source_.IncomingCapturedFrame(
4782 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4783 WaitForEncodedFrame(CurrentTimeMs());
4784 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4785 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4786 SizeIs(2));
4787 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4788 codec_width_);
4789 EXPECT_EQ(
4790 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4791 codec_height_);
4792
4793 video_source_.IncomingCapturedFrame(
4794 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4795 WaitForEncodedFrame(CurrentTimeMs());
4796 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4797 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4798 SizeIs(2));
4799 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4800 codec_width_ / 2);
4801 EXPECT_EQ(
4802 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4803 codec_height_ / 2);
4804
4805 video_stream_encoder_->Stop();
4806}
4807
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004808TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4809 // 2 TLs configured, temporal layers supported by encoder.
4810 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004811 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004812 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004813 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004814 fake_encoder_.SetTemporalLayersSupported(0, true);
4815
4816 // Bitrate allocated across temporal layers.
4817 const int kTl0Bps = kTargetBitrateBps *
4818 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004819 kNumTemporalLayers, /*temporal_id*/ 0,
4820 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004821 const int kTl1Bps = kTargetBitrateBps *
4822 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004823 kNumTemporalLayers, /*temporal_id*/ 1,
4824 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004825 VideoBitrateAllocation expected_bitrate;
4826 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4827 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4828
4829 VerifyAllocatedBitrate(expected_bitrate);
4830 video_stream_encoder_->Stop();
4831}
4832
4833TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4834 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004835 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004836 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004837 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004838 fake_encoder_.SetTemporalLayersSupported(0, false);
4839
4840 // Temporal layers not supported by the encoder.
4841 // Total bitrate should be at ti:0.
4842 VideoBitrateAllocation expected_bitrate;
4843 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4844
4845 VerifyAllocatedBitrate(expected_bitrate);
4846 video_stream_encoder_->Stop();
4847}
4848
4849TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004850 webrtc::test::ScopedFieldTrials field_trials(
4851 "WebRTC-Video-QualityScalerSettings/"
4852 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4853 // Reset encoder for field trials to take effect.
4854 ConfigureEncoder(video_encoder_config_.Copy());
4855
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004856 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004857 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004858 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004859 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004860 fake_encoder_.SetTemporalLayersSupported(0, true);
4861 fake_encoder_.SetTemporalLayersSupported(1, false);
4862
4863 const int kS0Bps = 150000;
4864 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004865 kS0Bps *
4866 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4867 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004868 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004869 kS0Bps *
4870 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4871 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004872 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4873 // Temporal layers not supported by si:1.
4874 VideoBitrateAllocation expected_bitrate;
4875 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4876 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4877 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4878
4879 VerifyAllocatedBitrate(expected_bitrate);
4880 video_stream_encoder_->Stop();
4881}
4882
Niels Möller7dc26b72017-12-06 10:27:48 +01004883TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4884 const int kFrameWidth = 1280;
4885 const int kFrameHeight = 720;
4886 const int kFramerate = 24;
4887
Henrik Boström381d1092020-05-12 18:49:07 +02004888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004889 DataRate::BitsPerSec(kTargetBitrateBps),
4890 DataRate::BitsPerSec(kTargetBitrateBps),
4891 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004892 test::FrameForwarder source;
4893 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004894 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004895
4896 // Insert a single frame, triggering initial configuration.
4897 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4898 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4899
4900 EXPECT_EQ(
4901 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4902 kDefaultFramerate);
4903
4904 // Trigger reconfigure encoder (without resetting the entire instance).
4905 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004906 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4907 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004908 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004909 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004910 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004911 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4912
4913 // Detector should be updated with fps limit from codec config.
4914 EXPECT_EQ(
4915 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4916 kFramerate);
4917
4918 // Trigger overuse, max framerate should be reduced.
4919 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4920 stats.input_frame_rate = kFramerate;
4921 stats_proxy_->SetMockStats(stats);
4922 video_stream_encoder_->TriggerCpuOveruse();
4923 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4924 int adapted_framerate =
4925 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4926 EXPECT_LT(adapted_framerate, kFramerate);
4927
4928 // Trigger underuse, max framerate should go back to codec configured fps.
4929 // Set extra low fps, to make sure it's actually reset, not just incremented.
4930 stats = stats_proxy_->GetStats();
4931 stats.input_frame_rate = adapted_framerate / 2;
4932 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004933 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4935 EXPECT_EQ(
4936 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4937 kFramerate);
4938
4939 video_stream_encoder_->Stop();
4940}
4941
4942TEST_F(VideoStreamEncoderTest,
4943 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
4944 const int kFrameWidth = 1280;
4945 const int kFrameHeight = 720;
4946 const int kLowFramerate = 15;
4947 const int kHighFramerate = 25;
4948
Henrik Boström381d1092020-05-12 18:49:07 +02004949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004950 DataRate::BitsPerSec(kTargetBitrateBps),
4951 DataRate::BitsPerSec(kTargetBitrateBps),
4952 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004953 test::FrameForwarder source;
4954 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004955 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004956
4957 // Trigger initial configuration.
4958 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02004959 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
4960 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004961 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01004962 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02004963 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02004964 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004965 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4966
4967 EXPECT_EQ(
4968 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4969 kLowFramerate);
4970
4971 // Trigger overuse, max framerate should be reduced.
4972 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4973 stats.input_frame_rate = kLowFramerate;
4974 stats_proxy_->SetMockStats(stats);
4975 video_stream_encoder_->TriggerCpuOveruse();
4976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4977 int adapted_framerate =
4978 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
4979 EXPECT_LT(adapted_framerate, kLowFramerate);
4980
4981 // Reconfigure the encoder with a new (higher max framerate), max fps should
4982 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02004983 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01004984 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4985 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02004986 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01004987 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4988
4989 EXPECT_EQ(
4990 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
4991 adapted_framerate);
4992
4993 // Trigger underuse, max framerate should go back to codec configured fps.
4994 stats = stats_proxy_->GetStats();
4995 stats.input_frame_rate = adapted_framerate;
4996 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02004997 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01004998 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4999 EXPECT_EQ(
5000 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5001 kHighFramerate);
5002
5003 video_stream_encoder_->Stop();
5004}
5005
mflodmancc3d4422017-08-03 08:27:51 -07005006TEST_F(VideoStreamEncoderTest,
5007 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005008 const int kFrameWidth = 1280;
5009 const int kFrameHeight = 720;
5010 const int kFramerate = 24;
5011
Henrik Boström381d1092020-05-12 18:49:07 +02005012 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005013 DataRate::BitsPerSec(kTargetBitrateBps),
5014 DataRate::BitsPerSec(kTargetBitrateBps),
5015 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005016 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005017 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005018 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005019
5020 // Trigger initial configuration.
5021 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005022 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5023 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005024 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005025 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005026 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005027 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005028 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005029
Niels Möller7dc26b72017-12-06 10:27:48 +01005030 EXPECT_EQ(
5031 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5032 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005033
5034 // Trigger overuse, max framerate should be reduced.
5035 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5036 stats.input_frame_rate = kFramerate;
5037 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005038 video_stream_encoder_->TriggerCpuOveruse();
5039 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005040 int adapted_framerate =
5041 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005042 EXPECT_LT(adapted_framerate, kFramerate);
5043
5044 // Change degradation preference to not enable framerate scaling. Target
5045 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005046 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005047 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005048 EXPECT_EQ(
5049 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5050 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005051
mflodmancc3d4422017-08-03 08:27:51 -07005052 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005053}
5054
mflodmancc3d4422017-08-03 08:27:51 -07005055TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005056 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005058 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5059 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5060 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005061 const int kWidth = 640;
5062 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005063
asaperssonfab67072017-04-04 05:51:49 -07005064 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005065
5066 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005067 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005068
5069 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005070 EXPECT_TRUE_WAIT(
5071 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005072
sprangc5d62e22017-04-02 23:53:04 -07005073 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005074
asaperssonfab67072017-04-04 05:51:49 -07005075 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005076 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005077 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005078
5079 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005080 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005081
Henrik Boström2671dac2020-05-19 16:29:09 +02005082 EXPECT_TRUE_WAIT(
5083 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005084
mflodmancc3d4422017-08-03 08:27:51 -07005085 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005086}
5087
mflodmancc3d4422017-08-03 08:27:51 -07005088TEST_F(VideoStreamEncoderTest,
5089 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005090 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005092 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5093 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5094 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005095 const int kWidth = 640;
5096 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005097
5098 // We expect the n initial frames to get dropped.
5099 int i;
5100 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005101 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005102 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005103 }
5104 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005105 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005106 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005107
5108 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005109 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005110
mflodmancc3d4422017-08-03 08:27:51 -07005111 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005112}
5113
mflodmancc3d4422017-08-03 08:27:51 -07005114TEST_F(VideoStreamEncoderTest,
5115 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005116 const int kWidth = 640;
5117 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005119 DataRate::BitsPerSec(kLowTargetBitrateBps),
5120 DataRate::BitsPerSec(kLowTargetBitrateBps),
5121 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005122
5123 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005124 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005125 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005126
asaperssonfab67072017-04-04 05:51:49 -07005127 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005128 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005129 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005130
mflodmancc3d4422017-08-03 08:27:51 -07005131 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005132}
5133
mflodmancc3d4422017-08-03 08:27:51 -07005134TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005135 const int kWidth = 640;
5136 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005137 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005138
5139 VideoEncoderConfig video_encoder_config;
5140 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5141 // Make format different, to force recreation of encoder.
5142 video_encoder_config.video_format.parameters["foo"] = "foo";
5143 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005144 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005146 DataRate::BitsPerSec(kLowTargetBitrateBps),
5147 DataRate::BitsPerSec(kLowTargetBitrateBps),
5148 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005149
kthelgasonb83797b2017-02-14 11:57:25 -08005150 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005151 video_stream_encoder_->SetSource(&video_source_,
5152 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005153
asaperssonfab67072017-04-04 05:51:49 -07005154 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005155 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005156 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005157
mflodmancc3d4422017-08-03 08:27:51 -07005158 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005159 fake_encoder_.SetQualityScaling(true);
5160}
5161
Åsa Persson139f4dc2019-08-02 09:29:58 +02005162TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5163 webrtc::test::ScopedFieldTrials field_trials(
5164 "WebRTC-Video-QualityScalerSettings/"
5165 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5166 // Reset encoder for field trials to take effect.
5167 ConfigureEncoder(video_encoder_config_.Copy());
5168 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5169 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5170 const int kWidth = 640;
5171 const int kHeight = 360;
5172
Henrik Boström381d1092020-05-12 18:49:07 +02005173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005174 DataRate::BitsPerSec(kTargetBitrateBps),
5175 DataRate::BitsPerSec(kTargetBitrateBps),
5176 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005177 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5178 // Frame should not be dropped.
5179 WaitForEncodedFrame(1);
5180
Henrik Boström381d1092020-05-12 18:49:07 +02005181 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005182 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5183 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5184 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005185 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5186 // Frame should not be dropped.
5187 WaitForEncodedFrame(2);
5188
Henrik Boström381d1092020-05-12 18:49:07 +02005189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005190 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5191 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5192 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005193 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5194 // Expect to drop this frame, the wait should time out.
5195 ExpectDroppedFrame();
5196
5197 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005198 EXPECT_TRUE_WAIT(
5199 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005200 video_stream_encoder_->Stop();
5201}
5202
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005203TEST_F(VideoStreamEncoderTest,
5204 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5205 webrtc::test::ScopedFieldTrials field_trials(
5206 "WebRTC-Video-QualityScalerSettings/"
5207 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5208 fake_encoder_.SetQualityScaling(false);
5209 ConfigureEncoder(video_encoder_config_.Copy());
5210 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5211 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5212 const int kWidth = 640;
5213 const int kHeight = 360;
5214
5215 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5216 DataRate::BitsPerSec(kTargetBitrateBps),
5217 DataRate::BitsPerSec(kTargetBitrateBps),
5218 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5219 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5220 // Frame should not be dropped.
5221 WaitForEncodedFrame(1);
5222
5223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5224 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5225 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5226 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5227 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5228 // Frame should not be dropped.
5229 WaitForEncodedFrame(2);
5230
5231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5232 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5233 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5234 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5235 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5236 // Not dropped since quality scaling is disabled.
5237 WaitForEncodedFrame(3);
5238
5239 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005240 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005241 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5242
5243 video_stream_encoder_->Stop();
5244}
5245
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005246TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5247 const int kLowTargetBitrateBps = 400000;
5248 // Set simulcast.
5249 ResetEncoder("VP8", 3, 1, 1, false);
5250 fake_encoder_.SetQualityScaling(true);
5251 const int kWidth = 1280;
5252 const int kHeight = 720;
5253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5254 DataRate::BitsPerSec(kLowTargetBitrateBps),
5255 DataRate::BitsPerSec(kLowTargetBitrateBps),
5256 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5257 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5258 // Frame should not be dropped.
5259 WaitForEncodedFrame(1);
5260
5261 // Trigger QVGA "singlecast"
5262 // Update the config.
5263 VideoEncoderConfig video_encoder_config;
5264 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5265 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005266 video_encoder_config.video_stream_factory =
5267 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5268 "VP8", /*max qp*/ 56, /*screencast*/ false,
5269 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005270 for (auto& layer : video_encoder_config.simulcast_layers) {
5271 layer.num_temporal_layers = 1;
5272 layer.max_framerate = kDefaultFramerate;
5273 }
5274 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5275 video_encoder_config.content_type =
5276 VideoEncoderConfig::ContentType::kRealtimeVideo;
5277
5278 video_encoder_config.simulcast_layers[0].active = true;
5279 video_encoder_config.simulcast_layers[1].active = false;
5280 video_encoder_config.simulcast_layers[2].active = false;
5281
5282 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5283 kMaxPayloadLength);
5284 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5285
5286 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5287 // Frame should not be dropped.
5288 WaitForEncodedFrame(2);
5289
5290 // Trigger HD "singlecast"
5291 video_encoder_config.simulcast_layers[0].active = false;
5292 video_encoder_config.simulcast_layers[1].active = false;
5293 video_encoder_config.simulcast_layers[2].active = true;
5294
5295 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5296 kMaxPayloadLength);
5297 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5298
5299 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5300 // Frame should be dropped because of initial frame drop.
5301 ExpectDroppedFrame();
5302
5303 // Expect the sink_wants to specify a scaled frame.
5304 EXPECT_TRUE_WAIT(
5305 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5306 video_stream_encoder_->Stop();
5307}
5308
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005309TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5310 const int kLowTargetBitrateBps = 400000;
5311 // Set simulcast.
5312 ResetEncoder("VP9", 1, 1, 3, false);
5313 fake_encoder_.SetQualityScaling(true);
5314 const int kWidth = 1280;
5315 const int kHeight = 720;
5316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5317 DataRate::BitsPerSec(kLowTargetBitrateBps),
5318 DataRate::BitsPerSec(kLowTargetBitrateBps),
5319 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5320 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5321 // Frame should not be dropped.
5322 WaitForEncodedFrame(1);
5323
5324 // Trigger QVGA "singlecast"
5325 // Update the config.
5326 VideoEncoderConfig video_encoder_config;
5327 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5328 &video_encoder_config);
5329 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5330 vp9_settings.numberOfSpatialLayers = 3;
5331 // Since only one layer is active - automatic resize should be enabled.
5332 vp9_settings.automaticResizeOn = true;
5333 video_encoder_config.encoder_specific_settings =
5334 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5335 vp9_settings);
5336 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5337 video_encoder_config.content_type =
5338 VideoEncoderConfig::ContentType::kRealtimeVideo;
5339 // Currently simulcast layers |active| flags are used to inidicate
5340 // which SVC layers are active.
5341 video_encoder_config.simulcast_layers.resize(3);
5342
5343 video_encoder_config.simulcast_layers[0].active = true;
5344 video_encoder_config.simulcast_layers[1].active = false;
5345 video_encoder_config.simulcast_layers[2].active = false;
5346
5347 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5348 kMaxPayloadLength);
5349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5350
5351 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5352 // Frame should not be dropped.
5353 WaitForEncodedFrame(2);
5354
5355 // Trigger HD "singlecast"
5356 video_encoder_config.simulcast_layers[0].active = false;
5357 video_encoder_config.simulcast_layers[1].active = false;
5358 video_encoder_config.simulcast_layers[2].active = true;
5359
5360 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5361 kMaxPayloadLength);
5362 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5363
5364 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5365 // Frame should be dropped because of initial frame drop.
5366 ExpectDroppedFrame();
5367
5368 // Expect the sink_wants to specify a scaled frame.
5369 EXPECT_TRUE_WAIT(
5370 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5371 video_stream_encoder_->Stop();
5372}
5373
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005374TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005375 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5376 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5377 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5378 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5379 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5380 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5381 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5382 fake_encoder_.SetResolutionBitrateLimits(
5383 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5384
5385 VideoEncoderConfig video_encoder_config;
5386 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5387 &video_encoder_config);
5388 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5389 vp9_settings.numberOfSpatialLayers = 3;
5390 // Since only one layer is active - automatic resize should be enabled.
5391 vp9_settings.automaticResizeOn = true;
5392 video_encoder_config.encoder_specific_settings =
5393 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5394 vp9_settings);
5395 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5396 video_encoder_config.content_type =
5397 VideoEncoderConfig::ContentType::kRealtimeVideo;
5398 // Simulcast layers are used to indicate which spatial layers are active.
5399 video_encoder_config.simulcast_layers.resize(3);
5400 video_encoder_config.simulcast_layers[0].active = false;
5401 video_encoder_config.simulcast_layers[1].active = true;
5402 video_encoder_config.simulcast_layers[2].active = false;
5403
5404 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5405 kMaxPayloadLength);
5406 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5407
5408 // The encoder bitrate limits for 360p should be used.
5409 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5410 EXPECT_FALSE(WaitForFrame(1000));
5411 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5412 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5413 VideoCodecType::kVideoCodecVP9);
5414 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5415 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5416 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5417 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5418 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5419 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5420 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5421 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5422
5423 // The encoder bitrate limits for 270p should be used.
5424 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5425 EXPECT_FALSE(WaitForFrame(1000));
5426 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5427 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5428 VideoCodecType::kVideoCodecVP9);
5429 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5430 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5431 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5432 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5433 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5434 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5435 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5436 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5437
5438 video_stream_encoder_->Stop();
5439}
5440
5441TEST_F(VideoStreamEncoderTest,
5442 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5443 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5444 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5445 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5446 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5447 fake_encoder_.SetResolutionBitrateLimits(
5448 {kEncoderLimits180p, kEncoderLimits720p});
5449
5450 VideoEncoderConfig video_encoder_config;
5451 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5452 &video_encoder_config);
5453 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5454 vp9_settings.numberOfSpatialLayers = 3;
5455 // Since only one layer is active - automatic resize should be enabled.
5456 vp9_settings.automaticResizeOn = true;
5457 video_encoder_config.encoder_specific_settings =
5458 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5459 vp9_settings);
5460 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5461 video_encoder_config.content_type =
5462 VideoEncoderConfig::ContentType::kRealtimeVideo;
5463 // Simulcast layers are used to indicate which spatial layers are active.
5464 video_encoder_config.simulcast_layers.resize(3);
5465 video_encoder_config.simulcast_layers[0].active = true;
5466 video_encoder_config.simulcast_layers[1].active = false;
5467 video_encoder_config.simulcast_layers[2].active = false;
5468
5469 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5470 kMaxPayloadLength);
5471 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5472
5473 // Limits not applied on lowest stream, limits for 180p should not be used.
5474 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5475 EXPECT_FALSE(WaitForFrame(1000));
5476 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5477 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5478 VideoCodecType::kVideoCodecVP9);
5479 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5480 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5481 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5482 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5483 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5484 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5485 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5486 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5487
5488 video_stream_encoder_->Stop();
5489}
5490
5491TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005492 InitialFrameDropActivatesWhenResolutionIncreases) {
5493 const int kWidth = 640;
5494 const int kHeight = 360;
5495
5496 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5497 DataRate::BitsPerSec(kTargetBitrateBps),
5498 DataRate::BitsPerSec(kTargetBitrateBps),
5499 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5500 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5501 // Frame should not be dropped.
5502 WaitForEncodedFrame(1);
5503
5504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5505 DataRate::BitsPerSec(kLowTargetBitrateBps),
5506 DataRate::BitsPerSec(kLowTargetBitrateBps),
5507 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5508 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5509 // Frame should not be dropped, bitrate not too low for frame.
5510 WaitForEncodedFrame(2);
5511
5512 // Incoming resolution increases.
5513 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5514 // Expect to drop this frame, bitrate too low for frame.
5515 ExpectDroppedFrame();
5516
5517 // Expect the sink_wants to specify a scaled frame.
5518 EXPECT_TRUE_WAIT(
5519 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5520 video_stream_encoder_->Stop();
5521}
5522
5523TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5524 const int kWidth = 640;
5525 const int kHeight = 360;
5526 // So that quality scaling doesn't happen by itself.
5527 fake_encoder_.SetQp(kQpHigh);
5528
5529 AdaptingFrameForwarder source(&time_controller_);
5530 source.set_adaptation_enabled(true);
5531 video_stream_encoder_->SetSource(
5532 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5533
5534 int timestamp = 1;
5535
5536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5537 DataRate::BitsPerSec(kTargetBitrateBps),
5538 DataRate::BitsPerSec(kTargetBitrateBps),
5539 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5540 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5541 WaitForEncodedFrame(timestamp);
5542 timestamp += 9000;
5543 // Long pause to disable all first BWE drop logic.
5544 AdvanceTime(TimeDelta::Millis(1000));
5545
5546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5547 DataRate::BitsPerSec(kLowTargetBitrateBps),
5548 DataRate::BitsPerSec(kLowTargetBitrateBps),
5549 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5550 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5551 // Not dropped frame, as initial frame drop is disabled by now.
5552 WaitForEncodedFrame(timestamp);
5553 timestamp += 9000;
5554 AdvanceTime(TimeDelta::Millis(100));
5555
5556 // Quality adaptation down.
5557 video_stream_encoder_->TriggerQualityLow();
5558
5559 // Adaptation has an effect.
5560 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5561 5000);
5562
5563 // Frame isn't dropped as initial frame dropper is disabled.
5564 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5565 WaitForEncodedFrame(timestamp);
5566 timestamp += 9000;
5567 AdvanceTime(TimeDelta::Millis(100));
5568
5569 // Quality adaptation up.
5570 video_stream_encoder_->TriggerQualityHigh();
5571
5572 // Adaptation has an effect.
5573 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5574 5000);
5575
5576 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5577 // Frame should not be dropped, as initial framedropper is off.
5578 WaitForEncodedFrame(timestamp);
5579
5580 video_stream_encoder_->Stop();
5581}
5582
Åsa Persson7f354f82021-02-04 15:52:15 +01005583TEST_F(VideoStreamEncoderTest,
5584 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5585 const int kMinStartBps360p = 222000;
5586 fake_encoder_.SetResolutionBitrateLimits(
5587 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5588 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5589 800000)});
5590
5591 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5592 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5593 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5594 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5595 0, 0, 0);
5596 // Frame should not be dropped, bitrate not too low for frame.
5597 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5598 WaitForEncodedFrame(1);
5599
5600 // Incoming resolution increases, initial frame drop activates.
5601 // Frame should be dropped, link allocation too low for frame.
5602 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5603 ExpectDroppedFrame();
5604
5605 // Expect sink_wants to specify a scaled frame.
5606 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5607 5000);
5608 video_stream_encoder_->Stop();
5609}
5610
5611TEST_F(VideoStreamEncoderTest,
5612 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5613 const int kMinStartBps360p = 222000;
5614 fake_encoder_.SetResolutionBitrateLimits(
5615 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5616 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5617 800000)});
5618
5619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5620 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5621 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5622 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5623 0, 0, 0);
5624 // Frame should not be dropped, bitrate not too low for frame.
5625 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5626 WaitForEncodedFrame(1);
5627
5628 // Incoming resolution increases, initial frame drop activates.
5629 // Frame should be dropped, link allocation not too low for frame.
5630 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5631 WaitForEncodedFrame(2);
5632
5633 video_stream_encoder_->Stop();
5634}
5635
Åsa Perssone644a032019-11-08 15:56:00 +01005636TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5637 webrtc::test::ScopedFieldTrials field_trials(
5638 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5639
5640 // Reset encoder for field trials to take effect.
5641 VideoEncoderConfig config = video_encoder_config_.Copy();
5642 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005643 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005644 ConfigureEncoder(std::move(config));
5645 fake_encoder_.SetQp(kQpLow);
5646
5647 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005648 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005649 source.set_adaptation_enabled(true);
5650 video_stream_encoder_->SetSource(&source,
5651 DegradationPreference::MAINTAIN_FRAMERATE);
5652
5653 // Start at low bitrate.
5654 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005655 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5656 DataRate::BitsPerSec(kLowBitrateBps),
5657 DataRate::BitsPerSec(kLowBitrateBps),
5658 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005659
5660 // Expect first frame to be dropped and resolution to be limited.
5661 const int kWidth = 1280;
5662 const int kHeight = 720;
5663 const int64_t kFrameIntervalMs = 100;
5664 int64_t timestamp_ms = kFrameIntervalMs;
5665 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5666 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005667 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5668 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005669
5670 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5672 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005673
5674 // Insert frames and advance |min_duration_ms|.
5675 for (size_t i = 1; i <= 10; i++) {
5676 timestamp_ms += kFrameIntervalMs;
5677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5678 WaitForEncodedFrame(timestamp_ms);
5679 }
5680 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5681 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5682
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005683 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005684
5685 // Insert frame should trigger high BW and release quality limitation.
5686 timestamp_ms += kFrameIntervalMs;
5687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5688 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005689 // The ramp-up code involves the adaptation queue, give it time to execute.
5690 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005691 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005692 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005693
5694 // Frame should not be adapted.
5695 timestamp_ms += kFrameIntervalMs;
5696 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5697 WaitForEncodedFrame(kWidth, kHeight);
5698 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5699
5700 video_stream_encoder_->Stop();
5701}
5702
mflodmancc3d4422017-08-03 08:27:51 -07005703TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005704 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005705 webrtc::test::ScopedFieldTrials field_trials(
5706 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005707 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005708 source.set_adaptation_enabled(true);
5709 video_stream_encoder_->SetSource(&source,
5710 DegradationPreference::MAINTAIN_FRAMERATE);
5711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5712 DataRate::BitsPerSec(kTargetBitrateBps),
5713 DataRate::BitsPerSec(kTargetBitrateBps),
5714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5715 fake_encoder_.SetQp(kQpHigh + 1);
5716 const int kWidth = 1280;
5717 const int kHeight = 720;
5718 const int64_t kFrameIntervalMs = 100;
5719 int64_t timestamp_ms = kFrameIntervalMs;
5720 for (size_t i = 1; i <= 100; i++) {
5721 timestamp_ms += kFrameIntervalMs;
5722 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5723 WaitForEncodedFrame(timestamp_ms);
5724 }
5725 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5726 // for the first time.
5727 // TODO(eshr): We should avoid these waits by using threads with simulated
5728 // time.
5729 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5730 2000 * 2.5 * 2);
5731 timestamp_ms += kFrameIntervalMs;
5732 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5733 WaitForEncodedFrame(timestamp_ms);
5734 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5735 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5736 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5737
5738 // Disable Quality scaling by turning off scaler on the encoder and
5739 // reconfiguring.
5740 fake_encoder_.SetQualityScaling(false);
5741 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5742 kMaxPayloadLength);
5743 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005744 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005745 // Since we turned off the quality scaler, the adaptations made by it are
5746 // removed.
5747 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5748 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5749
5750 video_stream_encoder_->Stop();
5751}
5752
5753TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005754 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5755 const int kTooSmallWidth = 10;
5756 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005758 DataRate::BitsPerSec(kTargetBitrateBps),
5759 DataRate::BitsPerSec(kTargetBitrateBps),
5760 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005761
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005762 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005763 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005764 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005765 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005766 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5768
5769 // Trigger adapt down, too small frame, expect no change.
5770 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005771 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005772 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005773 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07005774 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
5775 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
5776
mflodmancc3d4422017-08-03 08:27:51 -07005777 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07005778}
5779
mflodmancc3d4422017-08-03 08:27:51 -07005780TEST_F(VideoStreamEncoderTest,
5781 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07005782 const int kTooSmallWidth = 10;
5783 const int kTooSmallHeight = 10;
5784 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02005785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005786 DataRate::BitsPerSec(kTargetBitrateBps),
5787 DataRate::BitsPerSec(kTargetBitrateBps),
5788 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07005789
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005790 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07005791 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005792 video_stream_encoder_->SetSource(&source,
5793 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005794 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07005795 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
5797
5798 // Trigger adapt down, expect limited framerate.
5799 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005800 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07005801 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005802 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005803 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5804 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5805 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5806
5807 // Trigger adapt down, too small frame, expect no change.
5808 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07005809 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07005810 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005811 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07005812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5813 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
5814 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
5815
mflodmancc3d4422017-08-03 08:27:51 -07005816 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07005817}
5818
mflodmancc3d4422017-08-03 08:27:51 -07005819TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07005820 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02005821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005822 DataRate::BitsPerSec(kTargetBitrateBps),
5823 DataRate::BitsPerSec(kTargetBitrateBps),
5824 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02005825 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07005826 const int kFrameWidth = 1280;
5827 const int kFrameHeight = 720;
5828 video_source_.IncomingCapturedFrame(
5829 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005830 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07005831 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07005832}
5833
sprangb1ca0732017-02-01 08:38:12 -08005834// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07005835TEST_F(VideoStreamEncoderTest,
5836 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02005837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005838 DataRate::BitsPerSec(kTargetBitrateBps),
5839 DataRate::BitsPerSec(kTargetBitrateBps),
5840 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08005841
5842 const int kFrameWidth = 1280;
5843 const int kFrameHeight = 720;
5844 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07005845 // requested by
5846 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08005847 video_source_.set_adaptation_enabled(true);
5848
5849 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005850 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005851 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005852
5853 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07005854 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08005855 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005856 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005857 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08005858
asaperssonfab67072017-04-04 05:51:49 -07005859 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02005860 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08005861 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02005862 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005863 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08005864
mflodmancc3d4422017-08-03 08:27:51 -07005865 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08005866}
sprangfe627f32017-03-29 08:24:59 -07005867
mflodmancc3d4422017-08-03 08:27:51 -07005868TEST_F(VideoStreamEncoderTest,
5869 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07005870 const int kFrameWidth = 1280;
5871 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07005872
Henrik Boström381d1092020-05-12 18:49:07 +02005873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005874 DataRate::BitsPerSec(kTargetBitrateBps),
5875 DataRate::BitsPerSec(kTargetBitrateBps),
5876 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005878 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005879 video_source_.set_adaptation_enabled(true);
5880
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005881 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005882
5883 video_source_.IncomingCapturedFrame(
5884 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005885 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005886
5887 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07005888 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005889
5890 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07005891 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005892 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005893 video_source_.IncomingCapturedFrame(
5894 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005895 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07005896 }
5897
5898 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07005899 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005900 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005901 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005902 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005903 video_source_.IncomingCapturedFrame(
5904 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005905 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005906 ++num_frames_dropped;
5907 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005908 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005909 }
5910 }
5911
sprang4847ae62017-06-27 07:06:52 -07005912 // Add some slack to account for frames dropped by the frame dropper.
5913 const int kErrorMargin = 1;
5914 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005915 kErrorMargin);
5916
5917 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07005918 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07005919 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02005920 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005921 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005922 video_source_.IncomingCapturedFrame(
5923 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005924 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005925 ++num_frames_dropped;
5926 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005927 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005928 }
5929 }
sprang4847ae62017-06-27 07:06:52 -07005930 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07005931 kErrorMargin);
5932
5933 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02005934 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005935 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005936 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005937 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005938 video_source_.IncomingCapturedFrame(
5939 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005940 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005941 ++num_frames_dropped;
5942 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005943 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005944 }
5945 }
sprang4847ae62017-06-27 07:06:52 -07005946 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07005947 kErrorMargin);
5948
5949 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02005950 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07005951 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07005952 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07005953 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07005954 video_source_.IncomingCapturedFrame(
5955 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07005956 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07005957 ++num_frames_dropped;
5958 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01005959 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07005960 }
5961 }
5962 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
5963
mflodmancc3d4422017-08-03 08:27:51 -07005964 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07005965}
5966
mflodmancc3d4422017-08-03 08:27:51 -07005967TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07005968 const int kFramerateFps = 5;
5969 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07005970 const int kFrameWidth = 1280;
5971 const int kFrameHeight = 720;
5972
sprang4847ae62017-06-27 07:06:52 -07005973 // Reconfigure encoder with two temporal layers and screensharing, which will
5974 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02005975 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07005976
Henrik Boström381d1092020-05-12 18:49:07 +02005977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005978 DataRate::BitsPerSec(kTargetBitrateBps),
5979 DataRate::BitsPerSec(kTargetBitrateBps),
5980 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07005981 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005982 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07005983 video_source_.set_adaptation_enabled(true);
5984
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005985 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07005986
5987 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08005988 rtc::VideoSinkWants last_wants;
5989 do {
5990 last_wants = video_source_.sink_wants();
5991
sprangc5d62e22017-04-02 23:53:04 -07005992 // Insert frames to get a new fps estimate...
5993 for (int j = 0; j < kFramerateFps; ++j) {
5994 video_source_.IncomingCapturedFrame(
5995 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08005996 if (video_source_.last_sent_width()) {
5997 sink_.WaitForEncodedFrame(timestamp_ms);
5998 }
sprangc5d62e22017-04-02 23:53:04 -07005999 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006000 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006001 }
6002 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006003 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006004 } while (video_source_.sink_wants().max_framerate_fps <
6005 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006006
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006007 EXPECT_THAT(video_source_.sink_wants(),
6008 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006009
mflodmancc3d4422017-08-03 08:27:51 -07006010 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006011}
asaperssonf7e294d2017-06-13 23:25:22 -07006012
mflodmancc3d4422017-08-03 08:27:51 -07006013TEST_F(VideoStreamEncoderTest,
6014 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006015 const int kWidth = 1280;
6016 const int kHeight = 720;
6017 const int64_t kFrameIntervalMs = 150;
6018 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006020 DataRate::BitsPerSec(kTargetBitrateBps),
6021 DataRate::BitsPerSec(kTargetBitrateBps),
6022 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006023
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006024 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006025 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006026 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006027 video_stream_encoder_->SetSource(&source,
6028 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006029 timestamp_ms += kFrameIntervalMs;
6030 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006031 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006032 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006033 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6035 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6036
6037 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006038 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006039 timestamp_ms += kFrameIntervalMs;
6040 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006041 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006042 EXPECT_THAT(source.sink_wants(),
6043 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006044 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6045 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6046 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6047
6048 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006049 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006050 timestamp_ms += kFrameIntervalMs;
6051 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006052 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006053 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006054 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6055 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6056 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6057
6058 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006059 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006060 timestamp_ms += kFrameIntervalMs;
6061 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006062 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006063 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006064 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6065 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6066 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6067
6068 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006069 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006070 timestamp_ms += kFrameIntervalMs;
6071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006072 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006073 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006074 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6075 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6076 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6077
6078 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006079 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006080 timestamp_ms += kFrameIntervalMs;
6081 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006082 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006083 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006084 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6085 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6086 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6087
6088 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006089 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006090 timestamp_ms += kFrameIntervalMs;
6091 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006092 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006093 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006094 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6095 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6096 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6097
6098 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006099 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006100 timestamp_ms += kFrameIntervalMs;
6101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006102 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006103 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006104 rtc::VideoSinkWants last_wants = source.sink_wants();
6105 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6106 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6107 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6108
6109 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006110 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006111 timestamp_ms += kFrameIntervalMs;
6112 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006113 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006114 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006115 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6116 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6117 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6118
Evan Shrubsole64469032020-06-11 10:45:29 +02006119 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006120 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006121 timestamp_ms += kFrameIntervalMs;
6122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006123 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006124 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006125 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6126 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6127 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6128
6129 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006130 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006131 timestamp_ms += kFrameIntervalMs;
6132 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006133 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006134 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006135 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6136 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6137 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6138
6139 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006140 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006141 timestamp_ms += kFrameIntervalMs;
6142 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006143 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006144 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006145 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6146 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6147 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6148
6149 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006150 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006151 timestamp_ms += kFrameIntervalMs;
6152 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006153 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006154 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006155 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6156 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6157 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6158
6159 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006160 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006161 timestamp_ms += kFrameIntervalMs;
6162 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006163 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006164 EXPECT_THAT(source.sink_wants(), FpsMax());
6165 EXPECT_EQ(source.sink_wants().max_pixel_count,
6166 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6168 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6169 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6170
6171 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006172 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006173 timestamp_ms += kFrameIntervalMs;
6174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006175 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006176 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006177 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6179 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6180
Åsa Persson30ab0152019-08-27 12:22:33 +02006181 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006182 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006183 timestamp_ms += kFrameIntervalMs;
6184 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006185 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006186 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006187 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006188 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6190 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6191
6192 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006193 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006194 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006195 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6196
mflodmancc3d4422017-08-03 08:27:51 -07006197 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006198}
6199
mflodmancc3d4422017-08-03 08:27:51 -07006200TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006201 const int kWidth = 1280;
6202 const int kHeight = 720;
6203 const int64_t kFrameIntervalMs = 150;
6204 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006206 DataRate::BitsPerSec(kTargetBitrateBps),
6207 DataRate::BitsPerSec(kTargetBitrateBps),
6208 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006209
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006210 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006211 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006212 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006213 video_stream_encoder_->SetSource(&source,
6214 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006215 timestamp_ms += kFrameIntervalMs;
6216 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006217 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006218 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006219 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6220 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6221 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6222 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6223 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6224 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6225
6226 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006227 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006228 timestamp_ms += kFrameIntervalMs;
6229 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006230 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006231 EXPECT_THAT(source.sink_wants(),
6232 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6234 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6235 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6236 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6237 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6238 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6239
6240 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006241 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006242 timestamp_ms += kFrameIntervalMs;
6243 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006244 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006245 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6247 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6248 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6249 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6250 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6251 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6252
6253 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006254 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006255 timestamp_ms += kFrameIntervalMs;
6256 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006257 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006258 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006259 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006260 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6261 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6262 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6263 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6264 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6265
Evan Shrubsole64469032020-06-11 10:45:29 +02006266 // Trigger cpu adapt up, expect no change since QP is most limited.
6267 {
6268 // Store current sink wants since we expect no change and if there is no
6269 // change then last_wants() is not updated.
6270 auto previous_sink_wants = source.sink_wants();
6271 video_stream_encoder_->TriggerCpuUnderuse();
6272 timestamp_ms += kFrameIntervalMs;
6273 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6274 WaitForEncodedFrame(timestamp_ms);
6275 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6276 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6278 }
6279
6280 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6281 video_stream_encoder_->TriggerQualityHigh();
6282 timestamp_ms += kFrameIntervalMs;
6283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6284 WaitForEncodedFrame(timestamp_ms);
6285 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6288 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6289 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6290 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6291 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6292
6293 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6294 // expect increased resolution (960x540@30fps).
6295 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006296 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006297 timestamp_ms += kFrameIntervalMs;
6298 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006299 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006300 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006301 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6302 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6303 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6304 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6305 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006306 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006307
Evan Shrubsole64469032020-06-11 10:45:29 +02006308 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6309 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006310 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006311 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006312 timestamp_ms += kFrameIntervalMs;
6313 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006314 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006315 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006316 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006317 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6318 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6319 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6320 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6321 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006322 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006323
6324 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006325 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006326 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006327 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006328 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006329
mflodmancc3d4422017-08-03 08:27:51 -07006330 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006331}
6332
mflodmancc3d4422017-08-03 08:27:51 -07006333TEST_F(VideoStreamEncoderTest,
6334 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006335 const int kWidth = 640;
6336 const int kHeight = 360;
6337 const int kFpsLimit = 15;
6338 const int64_t kFrameIntervalMs = 150;
6339 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006341 DataRate::BitsPerSec(kTargetBitrateBps),
6342 DataRate::BitsPerSec(kTargetBitrateBps),
6343 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006344
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006345 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006346 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006347 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006348 video_stream_encoder_->SetSource(&source,
6349 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006350 timestamp_ms += kFrameIntervalMs;
6351 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006352 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006353 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006354 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6356 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6357 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6358 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6359 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6360
6361 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006363 timestamp_ms += kFrameIntervalMs;
6364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006365 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006366 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006367 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6368 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6369 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6370 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6371 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6372 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6373
6374 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006375 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006376 timestamp_ms += kFrameIntervalMs;
6377 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006378 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006379 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006380 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006381 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006382 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6383 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6384 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6385 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6386
Evan Shrubsole64469032020-06-11 10:45:29 +02006387 // Trigger cpu adapt up, expect no change because quality is most limited.
6388 {
6389 auto previous_sink_wants = source.sink_wants();
6390 // Store current sink wants since we expect no change ind if there is no
6391 // change then last__wants() is not updated.
6392 video_stream_encoder_->TriggerCpuUnderuse();
6393 timestamp_ms += kFrameIntervalMs;
6394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6395 WaitForEncodedFrame(timestamp_ms);
6396 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6397 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6398 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6399 }
6400
6401 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6402 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006403 timestamp_ms += kFrameIntervalMs;
6404 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006405 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006406 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006407 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6408 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6409 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006410 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6411 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6412 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006413
Evan Shrubsole64469032020-06-11 10:45:29 +02006414 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006415 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006416 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006417 timestamp_ms += kFrameIntervalMs;
6418 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006419 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006420 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6422 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6423 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6424 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6425 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006426 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006427
6428 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006429 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006430 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006431 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006432 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006433
mflodmancc3d4422017-08-03 08:27:51 -07006434 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006435}
6436
mflodmancc3d4422017-08-03 08:27:51 -07006437TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006438 const int kFrameWidth = 1920;
6439 const int kFrameHeight = 1080;
6440 // 3/4 of 1920.
6441 const int kAdaptedFrameWidth = 1440;
6442 // 3/4 of 1080 rounded down to multiple of 4.
6443 const int kAdaptedFrameHeight = 808;
6444 const int kFramerate = 24;
6445
Henrik Boström381d1092020-05-12 18:49:07 +02006446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006447 DataRate::BitsPerSec(kTargetBitrateBps),
6448 DataRate::BitsPerSec(kTargetBitrateBps),
6449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006450 // Trigger reconfigure encoder (without resetting the entire instance).
6451 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006452 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6453 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006454 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006455 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006456 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006457 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006458 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006459 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006460
6461 video_source_.set_adaptation_enabled(true);
6462
6463 video_source_.IncomingCapturedFrame(
6464 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006465 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006466
6467 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006468 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006469 video_source_.IncomingCapturedFrame(
6470 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006471 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006472
mflodmancc3d4422017-08-03 08:27:51 -07006473 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006474}
6475
mflodmancc3d4422017-08-03 08:27:51 -07006476TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006477 const int kFrameWidth = 1280;
6478 const int kFrameHeight = 720;
6479 const int kLowFps = 2;
6480 const int kHighFps = 30;
6481
Henrik Boström381d1092020-05-12 18:49:07 +02006482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006483 DataRate::BitsPerSec(kTargetBitrateBps),
6484 DataRate::BitsPerSec(kTargetBitrateBps),
6485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006486
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006487 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006488 max_framerate_ = kLowFps;
6489
6490 // Insert 2 seconds of 2fps video.
6491 for (int i = 0; i < kLowFps * 2; ++i) {
6492 video_source_.IncomingCapturedFrame(
6493 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6494 WaitForEncodedFrame(timestamp_ms);
6495 timestamp_ms += 1000 / kLowFps;
6496 }
6497
6498 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006500 DataRate::BitsPerSec(kTargetBitrateBps),
6501 DataRate::BitsPerSec(kTargetBitrateBps),
6502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006503 video_source_.IncomingCapturedFrame(
6504 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6505 WaitForEncodedFrame(timestamp_ms);
6506 timestamp_ms += 1000 / kLowFps;
6507
6508 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6509
6510 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006511 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006512 const int kFrameIntervalMs = 1000 / kHighFps;
6513 max_framerate_ = kHighFps;
6514 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6515 video_source_.IncomingCapturedFrame(
6516 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6517 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6518 // be dropped if the encoder hans't been updated with the new higher target
6519 // framerate yet, causing it to overshoot the target bitrate and then
6520 // suffering the wrath of the media optimizer.
6521 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6522 timestamp_ms += kFrameIntervalMs;
6523 }
6524
6525 // Don expect correct measurement just yet, but it should be higher than
6526 // before.
6527 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6528
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006530}
6531
mflodmancc3d4422017-08-03 08:27:51 -07006532TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006533 const int kFrameWidth = 1280;
6534 const int kFrameHeight = 720;
6535 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006536 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006537 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006538 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006539
Henrik Boström381d1092020-05-12 18:49:07 +02006540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006541 DataRate::BitsPerSec(kTargetBitrateBps),
6542 DataRate::BitsPerSec(kTargetBitrateBps),
6543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006544 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006545
6546 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006547 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006548 video_source_.IncomingCapturedFrame(
6549 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6550 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006551 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006552
6553 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6555 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6556 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006557
6558 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006559 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006560 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006561
Per Kjellanderdcef6412020-10-07 15:09:05 +02006562 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006563 video_source_.IncomingCapturedFrame(
6564 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6565 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006566 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006567
mflodmancc3d4422017-08-03 08:27:51 -07006568 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006569}
ilnik6b826ef2017-06-16 06:53:48 -07006570
Niels Möller4db138e2018-04-19 09:04:13 +02006571TEST_F(VideoStreamEncoderTest,
6572 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6573 const int kFrameWidth = 1280;
6574 const int kFrameHeight = 720;
6575 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006577 DataRate::BitsPerSec(kTargetBitrateBps),
6578 DataRate::BitsPerSec(kTargetBitrateBps),
6579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006580 video_source_.IncomingCapturedFrame(
6581 CreateFrame(1, kFrameWidth, kFrameHeight));
6582 WaitForEncodedFrame(1);
6583 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6584 .low_encode_usage_threshold_percent,
6585 default_options.low_encode_usage_threshold_percent);
6586 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6587 .high_encode_usage_threshold_percent,
6588 default_options.high_encode_usage_threshold_percent);
6589 video_stream_encoder_->Stop();
6590}
6591
6592TEST_F(VideoStreamEncoderTest,
6593 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6594 const int kFrameWidth = 1280;
6595 const int kFrameHeight = 720;
6596 CpuOveruseOptions hardware_options;
6597 hardware_options.low_encode_usage_threshold_percent = 150;
6598 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006599 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006600
Henrik Boström381d1092020-05-12 18:49:07 +02006601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006602 DataRate::BitsPerSec(kTargetBitrateBps),
6603 DataRate::BitsPerSec(kTargetBitrateBps),
6604 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006605 video_source_.IncomingCapturedFrame(
6606 CreateFrame(1, kFrameWidth, kFrameHeight));
6607 WaitForEncodedFrame(1);
6608 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6609 .low_encode_usage_threshold_percent,
6610 hardware_options.low_encode_usage_threshold_percent);
6611 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6612 .high_encode_usage_threshold_percent,
6613 hardware_options.high_encode_usage_threshold_percent);
6614 video_stream_encoder_->Stop();
6615}
6616
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006617TEST_F(VideoStreamEncoderTest,
6618 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6619 const int kFrameWidth = 1280;
6620 const int kFrameHeight = 720;
6621
6622 const CpuOveruseOptions default_options;
6623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6624 DataRate::BitsPerSec(kTargetBitrateBps),
6625 DataRate::BitsPerSec(kTargetBitrateBps),
6626 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6627 video_source_.IncomingCapturedFrame(
6628 CreateFrame(1, kFrameWidth, kFrameHeight));
6629 WaitForEncodedFrame(1);
6630 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6631 .low_encode_usage_threshold_percent,
6632 default_options.low_encode_usage_threshold_percent);
6633 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6634 .high_encode_usage_threshold_percent,
6635 default_options.high_encode_usage_threshold_percent);
6636
6637 CpuOveruseOptions hardware_options;
6638 hardware_options.low_encode_usage_threshold_percent = 150;
6639 hardware_options.high_encode_usage_threshold_percent = 200;
6640 fake_encoder_.SetIsHardwareAccelerated(true);
6641
6642 video_source_.IncomingCapturedFrame(
6643 CreateFrame(2, kFrameWidth, kFrameHeight));
6644 WaitForEncodedFrame(2);
6645
6646 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6647 .low_encode_usage_threshold_percent,
6648 hardware_options.low_encode_usage_threshold_percent);
6649 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6650 .high_encode_usage_threshold_percent,
6651 hardware_options.high_encode_usage_threshold_percent);
6652
6653 video_stream_encoder_->Stop();
6654}
6655
Niels Möller6bb5ab92019-01-11 11:11:10 +01006656TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6657 const int kFrameWidth = 320;
6658 const int kFrameHeight = 240;
6659 const int kFps = 30;
6660 const int kTargetBitrateBps = 120000;
6661 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6662
Henrik Boström381d1092020-05-12 18:49:07 +02006663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006664 DataRate::BitsPerSec(kTargetBitrateBps),
6665 DataRate::BitsPerSec(kTargetBitrateBps),
6666 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006667
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006668 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006669 max_framerate_ = kFps;
6670
6671 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6672 fake_encoder_.SimulateOvershoot(1.0);
6673 int num_dropped = 0;
6674 for (int i = 0; i < kNumFramesInRun; ++i) {
6675 video_source_.IncomingCapturedFrame(
6676 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6677 // Wait up to two frame durations for a frame to arrive.
6678 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6679 ++num_dropped;
6680 }
6681 timestamp_ms += 1000 / kFps;
6682 }
6683
Erik Språnga8d48ab2019-02-08 14:17:40 +01006684 // Framerate should be measured to be near the expected target rate.
6685 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6686
6687 // Frame drops should be within 5% of expected 0%.
6688 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006689
6690 // Make encoder produce frames at double the expected bitrate during 3 seconds
6691 // of video, verify number of drops. Rate needs to be slightly changed in
6692 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006693 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006694 const RateControlSettings trials =
6695 RateControlSettings::ParseFromFieldTrials();
6696 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006697 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006698 // frame dropping since the adjuter will try to just lower the target
6699 // bitrate rather than drop frames. If network headroom can be used, it
6700 // doesn't push back as hard so we don't need quite as much overshoot.
6701 // These numbers are unfortunately a bit magical but there's not trivial
6702 // way to algebraically infer them.
6703 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6704 overshoot_factor = 2.4;
6705 } else {
6706 overshoot_factor = 4.0;
6707 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006708 }
6709 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006711 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6712 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6713 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006714 num_dropped = 0;
6715 for (int i = 0; i < kNumFramesInRun; ++i) {
6716 video_source_.IncomingCapturedFrame(
6717 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6718 // Wait up to two frame durations for a frame to arrive.
6719 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6720 ++num_dropped;
6721 }
6722 timestamp_ms += 1000 / kFps;
6723 }
6724
Henrik Boström381d1092020-05-12 18:49:07 +02006725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006726 DataRate::BitsPerSec(kTargetBitrateBps),
6727 DataRate::BitsPerSec(kTargetBitrateBps),
6728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006729
6730 // Target framerate should be still be near the expected target, despite
6731 // the frame drops.
6732 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6733
6734 // Frame drops should be within 5% of expected 50%.
6735 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006736
6737 video_stream_encoder_->Stop();
6738}
6739
6740TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6741 const int kFrameWidth = 320;
6742 const int kFrameHeight = 240;
6743 const int kActualInputFps = 24;
6744 const int kTargetBitrateBps = 120000;
6745
6746 ASSERT_GT(max_framerate_, kActualInputFps);
6747
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006748 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006749 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006751 DataRate::BitsPerSec(kTargetBitrateBps),
6752 DataRate::BitsPerSec(kTargetBitrateBps),
6753 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006754
6755 // Insert 3 seconds of video, with an input fps lower than configured max.
6756 for (int i = 0; i < kActualInputFps * 3; ++i) {
6757 video_source_.IncomingCapturedFrame(
6758 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6759 // Wait up to two frame durations for a frame to arrive.
6760 WaitForEncodedFrame(timestamp_ms);
6761 timestamp_ms += 1000 / kActualInputFps;
6762 }
6763
6764 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6765
6766 video_stream_encoder_->Stop();
6767}
6768
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006769TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006770 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006772 DataRate::BitsPerSec(kTargetBitrateBps),
6773 DataRate::BitsPerSec(kTargetBitrateBps),
6774 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006775
6776 fake_encoder_.BlockNextEncode();
6777 video_source_.IncomingCapturedFrame(
6778 CreateFrameWithUpdatedPixel(1, nullptr, 0));
6779 WaitForEncodedFrame(1);
6780 // On the very first frame full update should be forced.
6781 rect = fake_encoder_.GetLastUpdateRect();
6782 EXPECT_EQ(rect.offset_x, 0);
6783 EXPECT_EQ(rect.offset_y, 0);
6784 EXPECT_EQ(rect.height, codec_height_);
6785 EXPECT_EQ(rect.width, codec_width_);
6786 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
6787 // call to ContinueEncode.
6788 video_source_.IncomingCapturedFrame(
6789 CreateFrameWithUpdatedPixel(2, nullptr, 1));
6790 ExpectDroppedFrame();
6791 video_source_.IncomingCapturedFrame(
6792 CreateFrameWithUpdatedPixel(3, nullptr, 10));
6793 ExpectDroppedFrame();
6794 fake_encoder_.ContinueEncode();
6795 WaitForEncodedFrame(3);
6796 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
6797 rect = fake_encoder_.GetLastUpdateRect();
6798 EXPECT_EQ(rect.offset_x, 1);
6799 EXPECT_EQ(rect.offset_y, 0);
6800 EXPECT_EQ(rect.width, 10);
6801 EXPECT_EQ(rect.height, 1);
6802
6803 video_source_.IncomingCapturedFrame(
6804 CreateFrameWithUpdatedPixel(4, nullptr, 0));
6805 WaitForEncodedFrame(4);
6806 // Previous frame was encoded, so no accumulation should happen.
6807 rect = fake_encoder_.GetLastUpdateRect();
6808 EXPECT_EQ(rect.offset_x, 0);
6809 EXPECT_EQ(rect.offset_y, 0);
6810 EXPECT_EQ(rect.width, 1);
6811 EXPECT_EQ(rect.height, 1);
6812
6813 video_stream_encoder_->Stop();
6814}
6815
Erik Språngd7329ca2019-02-21 21:19:53 +01006816TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02006817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006818 DataRate::BitsPerSec(kTargetBitrateBps),
6819 DataRate::BitsPerSec(kTargetBitrateBps),
6820 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006821
6822 // First frame is always keyframe.
6823 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6824 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01006825 EXPECT_THAT(
6826 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006827 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006828
6829 // Insert delta frame.
6830 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6831 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01006832 EXPECT_THAT(
6833 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006834 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006835
6836 // Request next frame be a key-frame.
6837 video_stream_encoder_->SendKeyFrame();
6838 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6839 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01006840 EXPECT_THAT(
6841 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006842 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006843
6844 video_stream_encoder_->Stop();
6845}
6846
6847TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
6848 // Setup simulcast with three streams.
6849 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006851 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6852 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
6853 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006854 // Wait for all three layers before triggering event.
6855 sink_.SetNumExpectedLayers(3);
6856
6857 // First frame is always keyframe.
6858 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6859 WaitForEncodedFrame(1);
6860 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006861 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
6862 VideoFrameType::kVideoFrameKey,
6863 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006864
6865 // Insert delta frame.
6866 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
6867 WaitForEncodedFrame(2);
6868 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006869 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
6870 VideoFrameType::kVideoFrameDelta,
6871 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006872
6873 // Request next frame be a key-frame.
6874 // Only first stream is configured to produce key-frame.
6875 video_stream_encoder_->SendKeyFrame();
6876 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
6877 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02006878
6879 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
6880 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01006881 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006882 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02006883 VideoFrameType::kVideoFrameKey,
6884 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006885
6886 video_stream_encoder_->Stop();
6887}
6888
6889TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
6890 // Configure internal source factory and setup test again.
6891 encoder_factory_.SetHasInternalSource(true);
6892 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006894 DataRate::BitsPerSec(kTargetBitrateBps),
6895 DataRate::BitsPerSec(kTargetBitrateBps),
6896 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01006897
6898 // Call encoder directly, simulating internal source where encoded frame
6899 // callback in VideoStreamEncoder is called despite no OnFrame().
6900 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
6901 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006902 EXPECT_THAT(
6903 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006904 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006905
Niels Möller8f7ce222019-03-21 15:43:58 +01006906 const std::vector<VideoFrameType> kDeltaFrame = {
6907 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01006908 // Need to set timestamp manually since manually for injected frame.
6909 VideoFrame frame = CreateFrame(101, nullptr);
6910 frame.set_timestamp(101);
6911 fake_encoder_.InjectFrame(frame, false);
6912 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006913 EXPECT_THAT(
6914 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006915 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006916
6917 // Request key-frame. The forces a dummy frame down into the encoder.
6918 fake_encoder_.ExpectNullFrame();
6919 video_stream_encoder_->SendKeyFrame();
6920 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01006921 EXPECT_THAT(
6922 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02006923 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01006924
6925 video_stream_encoder_->Stop();
6926}
Erik Språngb7cb7b52019-02-26 15:52:33 +01006927
6928TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
6929 // Configure internal source factory and setup test again.
6930 encoder_factory_.SetHasInternalSource(true);
6931 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02006932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006933 DataRate::BitsPerSec(kTargetBitrateBps),
6934 DataRate::BitsPerSec(kTargetBitrateBps),
6935 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006936
6937 int64_t timestamp = 1;
6938 EncodedImage image;
Niels Möller4d504c72019-06-18 15:56:56 +02006939 image.SetEncodedData(
6940 EncodedImageBuffer::Create(kTargetBitrateBps / kDefaultFramerate / 8));
Erik Språngb7cb7b52019-02-26 15:52:33 +01006941 image.capture_time_ms_ = ++timestamp;
6942 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
6943 const int64_t kEncodeFinishDelayMs = 10;
6944 image.timing_.encode_start_ms = timestamp;
6945 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
6946 fake_encoder_.InjectEncodedImage(image);
6947 // Wait for frame without incrementing clock.
6948 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
6949 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
6950 // capture timestamp should be kEncodeFinishDelayMs in the past.
6951 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006952 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01006953
6954 video_stream_encoder_->Stop();
6955}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006956
6957TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006958 // SPS contains VUI with restrictions on the maximum number of reordered
6959 // pictures, there is no need to rewrite the bitstream to enable faster
6960 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006961 ResetEncoder("H264", 1, 1, 1, false);
6962
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6964 DataRate::BitsPerSec(kTargetBitrateBps),
6965 DataRate::BitsPerSec(kTargetBitrateBps),
6966 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6967 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006968
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006969 fake_encoder_.SetEncodedImageData(
6970 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006971
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006972 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
6973 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006974
6975 EXPECT_THAT(sink_.GetLastEncodedImageData(),
6976 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006977
6978 video_stream_encoder_->Stop();
6979}
6980
6981TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006982 // SPS does not contain VUI, the bitstream is will be rewritten with added
6983 // VUI with restrictions on the maximum number of reordered pictures to
6984 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006985 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
6986 0x00, 0x00, 0x03, 0x03, 0xF4,
6987 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006988 ResetEncoder("H264", 1, 1, 1, false);
6989
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6991 DataRate::BitsPerSec(kTargetBitrateBps),
6992 DataRate::BitsPerSec(kTargetBitrateBps),
6993 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6994 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006995
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006996 fake_encoder_.SetEncodedImageData(
6997 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02006998
Mirta Dvornicic97910da2020-07-14 15:29:23 +02006999 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7000 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007001
7002 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7003 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007004
7005 video_stream_encoder_->Stop();
7006}
7007
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007008TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7009 const int kFrameWidth = 1280;
7010 const int kFrameHeight = 720;
7011 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7012
Henrik Boström381d1092020-05-12 18:49:07 +02007013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007014 DataRate::BitsPerSec(kTargetBitrateBps),
7015 DataRate::BitsPerSec(kTargetBitrateBps),
7016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7018
7019 // Insert a first video frame. It should be dropped because of downscale in
7020 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007021 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007022 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7023 frame.set_rotation(kVideoRotation_270);
7024 video_source_.IncomingCapturedFrame(frame);
7025
7026 ExpectDroppedFrame();
7027
7028 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007029 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007030 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7031 frame.set_rotation(kVideoRotation_90);
7032 video_source_.IncomingCapturedFrame(frame);
7033
7034 WaitForEncodedFrame(timestamp_ms);
7035 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7036
7037 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007038 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007039 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7040 frame.set_rotation(kVideoRotation_180);
7041 video_source_.IncomingCapturedFrame(frame);
7042
7043 WaitForEncodedFrame(timestamp_ms);
7044 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7045
7046 video_stream_encoder_->Stop();
7047}
7048
Erik Språng5056af02019-09-02 15:53:11 +02007049TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7050 const int kFrameWidth = 320;
7051 const int kFrameHeight = 180;
7052
7053 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007055 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7056 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7057 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007058 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007059 /*rtt_ms=*/0,
7060 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007061
7062 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007063 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007064 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7065 frame.set_rotation(kVideoRotation_270);
7066 video_source_.IncomingCapturedFrame(frame);
7067 WaitForEncodedFrame(timestamp_ms);
7068
7069 // Set a target rate below the minimum allowed by the codec settings.
7070 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007071 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7072 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007074 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007075 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007076 /*link_allocation=*/target_rate,
7077 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007078 /*rtt_ms=*/0,
7079 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007080 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7081
7082 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7083 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7084 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007085 DataRate allocation_sum =
7086 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007087 EXPECT_EQ(min_rate, allocation_sum);
7088 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7089
7090 video_stream_encoder_->Stop();
7091}
7092
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007093TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007095 DataRate::BitsPerSec(kTargetBitrateBps),
7096 DataRate::BitsPerSec(kTargetBitrateBps),
7097 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007098 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007099 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007100 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7101 WaitForEncodedFrame(1);
7102
7103 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7104 ASSERT_TRUE(prev_rate_settings.has_value());
7105 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7106 kDefaultFramerate);
7107
7108 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7109 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7110 timestamp_ms += 1000 / kDefaultFramerate;
7111 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7112 WaitForEncodedFrame(timestamp_ms);
7113 }
7114 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7115 kDefaultFramerate);
7116 // Capture larger frame to trigger a reconfigure.
7117 codec_height_ *= 2;
7118 codec_width_ *= 2;
7119 timestamp_ms += 1000 / kDefaultFramerate;
7120 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7121 WaitForEncodedFrame(timestamp_ms);
7122
7123 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7124 auto current_rate_settings =
7125 fake_encoder_.GetAndResetLastRateControlSettings();
7126 // Ensure we have actually reconfigured twice
7127 // The rate settings should have been set again even though
7128 // they haven't changed.
7129 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007130 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007131
7132 video_stream_encoder_->Stop();
7133}
7134
philipeld9cc8c02019-09-16 14:53:40 +02007135struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007136 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7137 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7138 MOCK_METHOD(void,
7139 RequestEncoderSwitch,
7140 (const webrtc::SdpVideoFormat& format),
7141 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007142};
7143
7144TEST_F(VideoStreamEncoderTest, BitrateEncoderSwitch) {
7145 constexpr int kDontCare = 100;
7146
7147 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7148 video_send_config_.encoder_settings.encoder_switch_request_callback =
7149 &switch_callback;
7150 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7151 encoder_config.codec_type = kVideoCodecVP8;
7152 webrtc::test::ScopedFieldTrials field_trial(
7153 "WebRTC-NetworkCondition-EncoderSwitch/"
7154 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
7155 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7156
7157 // Reset encoder for new configuration to take effect.
7158 ConfigureEncoder(std::move(encoder_config));
7159
7160 // Send one frame to trigger ReconfigureEncoder.
7161 video_source_.IncomingCapturedFrame(
7162 CreateFrame(kDontCare, kDontCare, kDontCare));
7163
7164 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01007165 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
7166 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02007167 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01007168 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02007169
Henrik Boström381d1092020-05-12 18:49:07 +02007170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007171 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7172 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7173 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipeld9cc8c02019-09-16 14:53:40 +02007174 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007175 /*rtt_ms=*/0,
7176 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007177 AdvanceTime(TimeDelta::Millis(0));
philipeld9cc8c02019-09-16 14:53:40 +02007178
7179 video_stream_encoder_->Stop();
7180}
7181
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01007182TEST_F(VideoStreamEncoderTest, VideoSuspendedNoEncoderSwitch) {
7183 constexpr int kDontCare = 100;
7184
7185 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7186 video_send_config_.encoder_settings.encoder_switch_request_callback =
7187 &switch_callback;
7188 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7189 encoder_config.codec_type = kVideoCodecVP8;
7190 webrtc::test::ScopedFieldTrials field_trial(
7191 "WebRTC-NetworkCondition-EncoderSwitch/"
7192 "codec_thresholds:VP8;100;-1|H264;-1;30000,"
7193 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7194
7195 // Reset encoder for new configuration to take effect.
7196 ConfigureEncoder(std::move(encoder_config));
7197
7198 // Send one frame to trigger ReconfigureEncoder.
7199 video_source_.IncomingCapturedFrame(
7200 CreateFrame(kDontCare, kDontCare, kDontCare));
7201
7202 using Config = EncoderSwitchRequestCallback::Config;
7203 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(_)))
7204 .Times(0);
7205
Henrik Boström381d1092020-05-12 18:49:07 +02007206 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Mirta Dvornicic5ed40cf2020-02-21 16:35:51 +01007207 /*target_bitrate=*/DataRate::KilobitsPerSec(0),
7208 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(0),
7209 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
7210 /*fraction_lost=*/0,
7211 /*rtt_ms=*/0,
7212 /*cwnd_reduce_ratio=*/0);
7213
7214 video_stream_encoder_->Stop();
7215}
7216
philipeld9cc8c02019-09-16 14:53:40 +02007217TEST_F(VideoStreamEncoderTest, ResolutionEncoderSwitch) {
7218 constexpr int kSufficientBitrateToNotDrop = 1000;
7219 constexpr int kHighRes = 500;
7220 constexpr int kLowRes = 100;
7221
7222 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7223 video_send_config_.encoder_settings.encoder_switch_request_callback =
7224 &switch_callback;
7225 webrtc::test::ScopedFieldTrials field_trial(
7226 "WebRTC-NetworkCondition-EncoderSwitch/"
7227 "codec_thresholds:VP8;120;-1|H264;-1;30000,"
7228 "to_codec:AV1,to_param:ping,to_value:pong,window:2.0/");
7229 VideoEncoderConfig encoder_config = video_encoder_config_.Copy();
7230 encoder_config.codec_type = kVideoCodecH264;
7231
7232 // Reset encoder for new configuration to take effect.
7233 ConfigureEncoder(std::move(encoder_config));
7234
7235 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7236 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7237 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007238 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007239 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7240 /*stable_target_bitrate=*/
7241 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7242 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipeld9cc8c02019-09-16 14:53:40 +02007243 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007244 /*rtt_ms=*/0,
7245 /*cwnd_reduce_ratio=*/0);
philipeld9cc8c02019-09-16 14:53:40 +02007246
7247 // Send one frame to trigger ReconfigureEncoder.
7248 video_source_.IncomingCapturedFrame(CreateFrame(1, kHighRes, kHighRes));
7249 WaitForEncodedFrame(1);
7250
7251 using Config = EncoderSwitchRequestCallback::Config;
philipel9b058032020-02-10 11:30:00 +01007252 EXPECT_CALL(switch_callback, RequestEncoderSwitch(Matcher<const Config&>(
7253 AllOf(Field(&Config::codec_name, "AV1"),
philipeld9cc8c02019-09-16 14:53:40 +02007254 Field(&Config::param, "ping"),
philipel9b058032020-02-10 11:30:00 +01007255 Field(&Config::value, "pong")))));
philipeld9cc8c02019-09-16 14:53:40 +02007256
7257 video_source_.IncomingCapturedFrame(CreateFrame(2, kLowRes, kLowRes));
7258 WaitForEncodedFrame(2);
7259
7260 video_stream_encoder_->Stop();
7261}
7262
philipel9b058032020-02-10 11:30:00 +01007263TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7264 constexpr int kDontCare = 100;
7265 StrictMock<MockEncoderSelector> encoder_selector;
7266 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7267 &fake_encoder_, &encoder_selector);
7268 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7269
7270 // Reset encoder for new configuration to take effect.
7271 ConfigureEncoder(video_encoder_config_.Copy());
7272
7273 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7274
7275 video_source_.IncomingCapturedFrame(
7276 CreateFrame(kDontCare, kDontCare, kDontCare));
7277 video_stream_encoder_->Stop();
7278
7279 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7280 // to it's factory, so in order for the encoder instance in the
7281 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7282 // reset the |video_stream_encoder_| here.
7283 video_stream_encoder_.reset();
7284}
7285
7286TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7287 constexpr int kDontCare = 100;
7288
7289 NiceMock<MockEncoderSelector> encoder_selector;
7290 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7291 video_send_config_.encoder_settings.encoder_switch_request_callback =
7292 &switch_callback;
7293 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7294 &fake_encoder_, &encoder_selector);
7295 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7296
7297 // Reset encoder for new configuration to take effect.
7298 ConfigureEncoder(video_encoder_config_.Copy());
7299
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007300 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007301 .WillByDefault(Return(SdpVideoFormat("AV1")));
7302 EXPECT_CALL(switch_callback,
7303 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7304 Field(&SdpVideoFormat::name, "AV1"))));
7305
Henrik Boström381d1092020-05-12 18:49:07 +02007306 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007307 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7308 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7309 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007310 /*fraction_lost=*/0,
7311 /*rtt_ms=*/0,
7312 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007313 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007314
7315 video_stream_encoder_->Stop();
7316}
7317
7318TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7319 constexpr int kSufficientBitrateToNotDrop = 1000;
7320 constexpr int kDontCare = 100;
7321
7322 NiceMock<MockVideoEncoder> video_encoder;
7323 NiceMock<MockEncoderSelector> encoder_selector;
7324 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7325 video_send_config_.encoder_settings.encoder_switch_request_callback =
7326 &switch_callback;
7327 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7328 &video_encoder, &encoder_selector);
7329 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7330
7331 // Reset encoder for new configuration to take effect.
7332 ConfigureEncoder(video_encoder_config_.Copy());
7333
7334 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7335 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7336 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007337 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007338 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7339 /*stable_target_bitrate=*/
7340 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7341 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007342 /*fraction_lost=*/0,
7343 /*rtt_ms=*/0,
7344 /*cwnd_reduce_ratio=*/0);
7345
7346 ON_CALL(video_encoder, Encode(_, _))
7347 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7348 ON_CALL(encoder_selector, OnEncoderBroken())
7349 .WillByDefault(Return(SdpVideoFormat("AV2")));
7350
7351 rtc::Event encode_attempted;
7352 EXPECT_CALL(switch_callback,
7353 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7354 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7355 EXPECT_EQ(format.name, "AV2");
7356 encode_attempted.Set();
7357 });
7358
7359 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7360 encode_attempted.Wait(3000);
7361
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007362 AdvanceTime(TimeDelta::Millis(0));
7363
philipel9b058032020-02-10 11:30:00 +01007364 video_stream_encoder_->Stop();
7365
7366 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7367 // to it's factory, so in order for the encoder instance in the
7368 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7369 // reset the |video_stream_encoder_| here.
7370 video_stream_encoder_.reset();
7371}
7372
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007373TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007374 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007375 const int kFrameWidth = 320;
7376 const int kFrameHeight = 180;
7377
7378 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007379 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007381 /*target_bitrate=*/rate,
7382 /*stable_target_bitrate=*/rate,
7383 /*link_allocation=*/rate,
7384 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007385 /*rtt_ms=*/0,
7386 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007387
7388 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007389 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007390 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7391 frame.set_rotation(kVideoRotation_270);
7392 video_source_.IncomingCapturedFrame(frame);
7393 WaitForEncodedFrame(timestamp_ms);
7394 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7395
7396 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007397 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007399 /*target_bitrate=*/new_stable_rate,
7400 /*stable_target_bitrate=*/new_stable_rate,
7401 /*link_allocation=*/rate,
7402 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007403 /*rtt_ms=*/0,
7404 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007405 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7406 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7407 video_stream_encoder_->Stop();
7408}
7409
7410TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007411 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007412 const int kFrameWidth = 320;
7413 const int kFrameHeight = 180;
7414
7415 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007416 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007418 /*target_bitrate=*/rate,
7419 /*stable_target_bitrate=*/rate,
7420 /*link_allocation=*/rate,
7421 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007422 /*rtt_ms=*/0,
7423 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007424
7425 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007426 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007427 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7428 frame.set_rotation(kVideoRotation_270);
7429 video_source_.IncomingCapturedFrame(frame);
7430 WaitForEncodedFrame(timestamp_ms);
7431 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7432
7433 // Set a higher target rate without changing the link_allocation. Should not
7434 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007435 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007436 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007437 /*target_bitrate=*/rate,
7438 /*stable_target_bitrate=*/new_stable_rate,
7439 /*link_allocation=*/rate,
7440 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007441 /*rtt_ms=*/0,
7442 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007443 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7444 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7445 video_stream_encoder_->Stop();
7446}
7447
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007448TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7449 test::ScopedFieldTrials field_trials(
7450 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7451 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7452 const int kFramerateFps = 30;
7453 const int kWidth = 1920;
7454 const int kHeight = 1080;
7455 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7456 // Works on screenshare mode.
7457 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7458 // We rely on the automatic resolution adaptation, but we handle framerate
7459 // adaptation manually by mocking the stats proxy.
7460 video_source_.set_adaptation_enabled(true);
7461
7462 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007463 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007464 DataRate::BitsPerSec(kTargetBitrateBps),
7465 DataRate::BitsPerSec(kTargetBitrateBps),
7466 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007467 video_stream_encoder_->SetSource(&video_source_,
7468 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007469 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007470
7471 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7472 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7473
7474 // Pass enough frames with the full update to trigger animation detection.
7475 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007476 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007477 frame.set_ntp_time_ms(timestamp_ms);
7478 frame.set_timestamp_us(timestamp_ms * 1000);
7479 video_source_.IncomingCapturedFrame(frame);
7480 WaitForEncodedFrame(timestamp_ms);
7481 }
7482
7483 // Resolution should be limited.
7484 rtc::VideoSinkWants expected;
7485 expected.max_framerate_fps = kFramerateFps;
7486 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007487 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007488
7489 // Pass one frame with no known update.
7490 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007491 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007492 frame.set_ntp_time_ms(timestamp_ms);
7493 frame.set_timestamp_us(timestamp_ms * 1000);
7494 frame.clear_update_rect();
7495
7496 video_source_.IncomingCapturedFrame(frame);
7497 WaitForEncodedFrame(timestamp_ms);
7498
7499 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007500 EXPECT_THAT(video_source_.sink_wants(),
7501 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007502
7503 video_stream_encoder_->Stop();
7504}
7505
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007506TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7507 const int kWidth = 720; // 540p adapted down.
7508 const int kHeight = 405;
7509 const int kNumFrames = 3;
7510 // Works on screenshare mode.
7511 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7512 /*num_spatial_layers=*/2, /*screenshare=*/true);
7513
7514 video_source_.set_adaptation_enabled(true);
7515
7516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7517 DataRate::BitsPerSec(kTargetBitrateBps),
7518 DataRate::BitsPerSec(kTargetBitrateBps),
7519 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7520
7521 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7522
7523 // Pass enough frames with the full update to trigger animation detection.
7524 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007525 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007526 frame.set_ntp_time_ms(timestamp_ms);
7527 frame.set_timestamp_us(timestamp_ms * 1000);
7528 video_source_.IncomingCapturedFrame(frame);
7529 WaitForEncodedFrame(timestamp_ms);
7530 }
7531
7532 video_stream_encoder_->Stop();
7533}
7534
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007535TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7536 const float downscale_factors[] = {4.0, 2.0, 1.0};
7537 const int number_layers =
7538 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7539 VideoEncoderConfig config;
7540 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7541 for (int i = 0; i < number_layers; ++i) {
7542 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7543 config.simulcast_layers[i].active = true;
7544 }
7545 config.video_stream_factory =
7546 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7547 "VP8", /*max qp*/ 56, /*screencast*/ false,
7548 /*screenshare enabled*/ false);
7549 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7550 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7551 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7552 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7553
7554 // First initialization.
7555 // Encoder should be initialized. Next frame should be key frame.
7556 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7557 sink_.SetNumExpectedLayers(number_layers);
7558 int64_t timestamp_ms = kFrameIntervalMs;
7559 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7560 WaitForEncodedFrame(timestamp_ms);
7561 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7562 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7563 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7564 VideoFrameType::kVideoFrameKey,
7565 VideoFrameType::kVideoFrameKey}));
7566
7567 // Disable top layer.
7568 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7569 config.simulcast_layers[number_layers - 1].active = false;
7570 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7571 sink_.SetNumExpectedLayers(number_layers - 1);
7572 timestamp_ms += kFrameIntervalMs;
7573 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7574 WaitForEncodedFrame(timestamp_ms);
7575 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7576 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7577 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7578 VideoFrameType::kVideoFrameDelta,
7579 VideoFrameType::kVideoFrameDelta}));
7580
7581 // Re-enable top layer.
7582 // Encoder should be re-initialized. Next frame should be key frame.
7583 config.simulcast_layers[number_layers - 1].active = true;
7584 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7585 sink_.SetNumExpectedLayers(number_layers);
7586 timestamp_ms += kFrameIntervalMs;
7587 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7588 WaitForEncodedFrame(timestamp_ms);
7589 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7590 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7591 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7592 VideoFrameType::kVideoFrameKey,
7593 VideoFrameType::kVideoFrameKey}));
7594
7595 // Top layer max rate change.
7596 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7597 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7598 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7599 sink_.SetNumExpectedLayers(number_layers);
7600 timestamp_ms += kFrameIntervalMs;
7601 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7602 WaitForEncodedFrame(timestamp_ms);
7603 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7604 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7605 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7606 VideoFrameType::kVideoFrameDelta,
7607 VideoFrameType::kVideoFrameDelta}));
7608
7609 // Top layer resolution change.
7610 // Encoder should be re-initialized. Next frame should be key frame.
7611 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7612 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7613 sink_.SetNumExpectedLayers(number_layers);
7614 timestamp_ms += kFrameIntervalMs;
7615 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7616 WaitForEncodedFrame(timestamp_ms);
7617 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7618 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7619 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7620 VideoFrameType::kVideoFrameKey,
7621 VideoFrameType::kVideoFrameKey}));
7622 video_stream_encoder_->Stop();
7623}
7624
Henrik Boström1124ed12021-02-25 10:30:39 +01007625TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7626 const int kFrameWidth = 1280;
7627 const int kFrameHeight = 720;
7628
7629 SetUp();
7630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7631 DataRate::BitsPerSec(kTargetBitrateBps),
7632 DataRate::BitsPerSec(kTargetBitrateBps),
7633 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7634
7635 // Capturing a frame should reconfigure the encoder and expose the encoder
7636 // resolution, which is the same as the input frame.
7637 int64_t timestamp_ms = kFrameIntervalMs;
7638 video_source_.IncomingCapturedFrame(
7639 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7640 WaitForEncodedFrame(timestamp_ms);
7641 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7642 EXPECT_THAT(video_source_.sink_wants().resolutions,
7643 ::testing::ElementsAreArray(
7644 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7645
7646 video_stream_encoder_->Stop();
7647}
7648
7649TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7650 // Pick downscale factors such that we never encode at full resolution - this
7651 // is an interesting use case. The frame resolution influences the encoder
7652 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7653 // encoder should not ask for the frame resolution. This allows video frames
7654 // to have the appearence of one resolution but optimize its internal buffers
7655 // for what is actually encoded.
7656 const size_t kNumSimulcastLayers = 3u;
7657 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7658 const int kFrameWidth = 1280;
7659 const int kFrameHeight = 720;
7660 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7661 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7662 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7663 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7664 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7665 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7666
7667 VideoEncoderConfig config;
7668 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7669 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7670 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7671 config.simulcast_layers[i].active = true;
7672 }
7673 config.video_stream_factory =
7674 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7675 "VP8", /*max qp*/ 56, /*screencast*/ false,
7676 /*screenshare enabled*/ false);
7677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7678 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7679 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7680 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7681
7682 // Capture a frame with all layers active.
7683 int64_t timestamp_ms = kFrameIntervalMs;
7684 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7685 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7686 video_source_.IncomingCapturedFrame(
7687 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7688 WaitForEncodedFrame(timestamp_ms);
7689 // Expect encoded resolutions to match the expected simulcast layers.
7690 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7691 EXPECT_THAT(
7692 video_source_.sink_wants().resolutions,
7693 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7694
7695 // Capture a frame with one of the layers inactive.
7696 timestamp_ms += kFrameIntervalMs;
7697 config.simulcast_layers[2].active = false;
7698 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7699 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7700 video_source_.IncomingCapturedFrame(
7701 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7702 WaitForEncodedFrame(timestamp_ms);
7703
7704 // Expect encoded resolutions to match the expected simulcast layers.
7705 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7706 EXPECT_THAT(video_source_.sink_wants().resolutions,
7707 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7708
7709 // Capture a frame with all but one layer turned off.
7710 timestamp_ms += kFrameIntervalMs;
7711 config.simulcast_layers[1].active = false;
7712 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7713 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7714 video_source_.IncomingCapturedFrame(
7715 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7716 WaitForEncodedFrame(timestamp_ms);
7717
7718 // Expect encoded resolutions to match the expected simulcast layers.
7719 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7720 EXPECT_THAT(video_source_.sink_wants().resolutions,
7721 ::testing::ElementsAreArray({kLayer0Size}));
7722
7723 video_stream_encoder_->Stop();
7724}
7725
perkj26091b12016-09-01 01:17:40 -07007726} // namespace webrtc