blob: 32d4f94e8c18d84a422527fda8744b3aacb94eff [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010022#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020023#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010025#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080026#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020028#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020029#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010030#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010031#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020032#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020033#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010034#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020035#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010036#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020037#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070038#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020040#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010041#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
42#include "modules/video_coding/codecs/h264/include/h264.h"
43#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
44#include "modules/video_coding/codecs/vp8/include/vp8.h"
45#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020046#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020047#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020048#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010049#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020050#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010051#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020052#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020053#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080054#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020055#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010056#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020057#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "test/encoder_settings.h"
59#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020060#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010061#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/gmock.h"
63#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010064#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020065#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020066#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010067#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020068#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070069
70namespace webrtc {
71
sprang57c2fff2017-01-16 06:24:02 -080072using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020074using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020075using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Ge;
78using ::testing::Gt;
79using ::testing::Le;
80using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010081using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010082using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010083using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010084using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010085using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010086using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020087using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080088
perkj803d97f2016-11-01 11:45:46 -070089namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020090const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010091const int kQpLow = 1;
92const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020093const int kMinFramerateFps = 2;
94const int kMinBalancedFramerateFps = 7;
95const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080096const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020097const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
98const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
99const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
100const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800101const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700102const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200103const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200104const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200105const VideoEncoder::ResolutionBitrateLimits
106 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
107const VideoEncoder::ResolutionBitrateLimits
108 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800109
Asa Persson606d3cb2021-10-04 10:07:11 +0200110uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200111 0x00, 0x00, 0x03, 0x03, 0xF4,
112 0x05, 0x03, 0xC7, 0xE0, 0x1B,
113 0x41, 0x10, 0x8D, 0x00};
114
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100115const uint8_t kCodedFrameVp8Qp25[] = {
116 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
117 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
118 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
119
perkj803d97f2016-11-01 11:45:46 -0700120class TestBuffer : public webrtc::I420Buffer {
121 public:
122 TestBuffer(rtc::Event* event, int width, int height)
123 : I420Buffer(width, height), event_(event) {}
124
125 private:
126 friend class rtc::RefCountedObject<TestBuffer>;
127 ~TestBuffer() override {
128 if (event_)
129 event_->Set();
130 }
131 rtc::Event* const event_;
132};
133
Henrik Boström56db9ff2021-03-24 09:06:45 +0100134// A fake native buffer that can't be converted to I420. Upon scaling, it
135// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700136class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
137 public:
138 FakeNativeBuffer(rtc::Event* event, int width, int height)
139 : event_(event), width_(width), height_(height) {}
140 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
141 int width() const override { return width_; }
142 int height() const override { return height_; }
143 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
144 return nullptr;
145 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100146 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
147 int offset_x,
148 int offset_y,
149 int crop_width,
150 int crop_height,
151 int scaled_width,
152 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200153 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
154 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100155 }
Noah Richards51db4212019-06-12 06:59:12 -0700156
157 private:
158 friend class rtc::RefCountedObject<FakeNativeBuffer>;
159 ~FakeNativeBuffer() override {
160 if (event_)
161 event_->Set();
162 }
163 rtc::Event* const event_;
164 const int width_;
165 const int height_;
166};
167
Evan Shrubsole895556e2020-10-05 09:15:13 +0200168// A fake native buffer that is backed by an NV12 buffer.
169class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
170 public:
171 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
172 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
173
174 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
175 int width() const override { return nv12_buffer_->width(); }
176 int height() const override { return nv12_buffer_->height(); }
177 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
178 return nv12_buffer_->ToI420();
179 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200180 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
181 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
182 if (absl::c_find(types, Type::kNV12) != types.end()) {
183 return nv12_buffer_;
184 }
185 return nullptr;
186 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200187 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
188
189 private:
190 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
191 ~FakeNV12NativeBuffer() override {
192 if (event_)
193 event_->Set();
194 }
195 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
196 rtc::Event* const event_;
197};
198
Niels Möller7dc26b72017-12-06 10:27:48 +0100199class CpuOveruseDetectorProxy : public OveruseFrameDetector {
200 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200201 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
202 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200203 last_target_framerate_fps_(-1),
204 framerate_updated_event_(true /* manual_reset */,
205 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100206 virtual ~CpuOveruseDetectorProxy() {}
207
208 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200209 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100210 last_target_framerate_fps_ = framerate_fps;
211 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200212 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100213 }
214
215 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200216 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100217 return last_target_framerate_fps_;
218 }
219
Niels Möller4db138e2018-04-19 09:04:13 +0200220 CpuOveruseOptions GetOptions() { return options_; }
221
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
223
Niels Möller7dc26b72017-12-06 10:27:48 +0100224 private:
Markus Handella3765182020-07-08 13:13:32 +0200225 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100226 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200227 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100228};
229
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230class FakeVideoSourceRestrictionsListener
231 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200232 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200233 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200234 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200235 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200236 RTC_DCHECK(was_restrictions_updated_);
237 }
238
239 rtc::Event* restrictions_updated_event() {
240 return &restrictions_updated_event_;
241 }
242
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200243 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200244 void OnVideoSourceRestrictionsUpdated(
245 VideoSourceRestrictions restrictions,
246 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200247 rtc::scoped_refptr<Resource> reason,
248 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200249 was_restrictions_updated_ = true;
250 restrictions_updated_event_.Set();
251 }
252
253 private:
254 bool was_restrictions_updated_;
255 rtc::Event restrictions_updated_event_;
256};
257
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200258auto WantsFps(Matcher<int> fps_matcher) {
259 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
260 fps_matcher);
261}
262
263auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
264 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
265 AllOf(max_pixel_matcher, Gt(0)));
266}
267
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200268auto ResolutionMax() {
269 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200270 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200271 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
272 Eq(absl::nullopt)));
273}
274
275auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200276 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200277}
278
279auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200280 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200281}
282
283auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200284 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200285}
286
287auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200288 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200289}
290
291auto FpsMaxResolutionMax() {
292 return AllOf(FpsMax(), ResolutionMax());
293}
294
295auto UnlimitedSinkWants() {
296 return AllOf(FpsUnlimited(), ResolutionMax());
297}
298
299auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
300 Matcher<int> fps_range_matcher;
301
302 if (last_frame_pixels <= 320 * 240) {
303 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200304 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200305 fps_range_matcher = AllOf(Ge(10), Le(15));
306 } else if (last_frame_pixels <= 640 * 480) {
307 fps_range_matcher = Ge(15);
308 } else {
309 fps_range_matcher = Eq(kDefaultFramerate);
310 }
311 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
312 fps_range_matcher);
313}
314
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200315auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
317 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
318}
319
320auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
321 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
322}
323
324auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
325 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
326}
327
328auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
341}
342
343auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
345 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
346}
347
mflodmancc3d4422017-08-03 08:27:51 -0700348class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700349 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100350 VideoStreamEncoderUnderTest(
351 TimeController* time_controller,
352 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
353 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
354 encoder_queue,
355 SendStatisticsProxy* stats_proxy,
356 const VideoStreamEncoderSettings& settings,
357 VideoStreamEncoder::BitrateAllocationCallbackType
358 allocation_callback_type)
359 : VideoStreamEncoder(time_controller->GetClock(),
360 1 /* number_of_cores */,
361 stats_proxy,
362 settings,
363 std::unique_ptr<OveruseFrameDetector>(
364 overuse_detector_proxy_ =
365 new CpuOveruseDetectorProxy(stats_proxy)),
366 std::move(cadence_adapter),
367 std::move(encoder_queue),
368 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200369 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200370 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200371 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200372 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200373 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200374 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200375 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200376 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100377 }
perkj803d97f2016-11-01 11:45:46 -0700378
Henrik Boström381d1092020-05-12 18:49:07 +0200379 void SetSourceAndWaitForRestrictionsUpdated(
380 rtc::VideoSourceInterface<VideoFrame>* source,
381 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200382 FakeVideoSourceRestrictionsListener listener;
383 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200384 SetSource(source, degradation_preference);
385 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200386 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200387 }
388
389 void SetSourceAndWaitForFramerateUpdated(
390 rtc::VideoSourceInterface<VideoFrame>* source,
391 const DegradationPreference& degradation_preference) {
392 overuse_detector_proxy_->framerate_updated_event()->Reset();
393 SetSource(source, degradation_preference);
394 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
395 }
396
397 void OnBitrateUpdatedAndWaitForManagedResources(
398 DataRate target_bitrate,
399 DataRate stable_target_bitrate,
400 DataRate link_allocation,
401 uint8_t fraction_lost,
402 int64_t round_trip_time_ms,
403 double cwnd_reduce_ratio) {
404 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
405 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
406 // Bitrate is updated on the encoder queue.
407 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200408 }
409
kthelgason2fc52542017-03-03 00:24:41 -0800410 // This is used as a synchronisation mechanism, to make sure that the
411 // encoder queue is not blocked before we start sending it frames.
412 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100413 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800414 }
415
Henrik Boström91aa7322020-04-28 12:24:33 +0200416 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200417 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200419 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200420 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 event.Set();
422 });
423 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100424 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200425 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200426
Henrik Boström91aa7322020-04-28 12:24:33 +0200427 void TriggerCpuUnderuse() {
428 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200429 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200430 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200431 event.Set();
432 });
433 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100434 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200435 }
kthelgason876222f2016-11-29 01:44:11 -0800436
Henrik Boström91aa7322020-04-28 12:24:33 +0200437 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200438 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200439 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200440 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200441 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 event.Set();
443 });
444 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100445 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200446 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200447 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200450 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 event.Set();
452 });
453 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100454 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 }
456
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200457 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100458 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200459 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
460 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200461 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700462};
463
Noah Richards51db4212019-06-12 06:59:12 -0700464// Simulates simulcast behavior and makes highest stream resolutions divisible
465// by 4.
466class CroppingVideoStreamFactory
467 : public VideoEncoderConfig::VideoStreamFactoryInterface {
468 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200469 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700470
471 private:
472 std::vector<VideoStream> CreateEncoderStreams(
473 int width,
474 int height,
475 const VideoEncoderConfig& encoder_config) override {
476 std::vector<VideoStream> streams = test::CreateVideoStreams(
477 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700478 return streams;
479 }
Noah Richards51db4212019-06-12 06:59:12 -0700480};
481
sprangb1ca0732017-02-01 08:38:12 -0800482class AdaptingFrameForwarder : public test::FrameForwarder {
483 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200484 explicit AdaptingFrameForwarder(TimeController* time_controller)
485 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700486 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800487
488 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200489 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800490 adaptation_enabled_ = enabled;
491 }
492
asaperssonfab67072017-04-04 05:51:49 -0700493 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200494 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800495 return adaptation_enabled_;
496 }
497
Henrik Boström1124ed12021-02-25 10:30:39 +0100498 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
499 // the resolution or frame rate was different than it is currently. If
500 // something else is modified, such as encoder resolutions, but the resolution
501 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700502 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200503 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700504 return last_wants_;
505 }
506
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200507 absl::optional<int> last_sent_width() const { return last_width_; }
508 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800509
sprangb1ca0732017-02-01 08:38:12 -0800510 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100512 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200513
sprangb1ca0732017-02-01 08:38:12 -0800514 int cropped_width = 0;
515 int cropped_height = 0;
516 int out_width = 0;
517 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700518 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000519 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
520 << "w=" << video_frame.width()
521 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700522 if (adapter_.AdaptFrameResolution(
523 video_frame.width(), video_frame.height(),
524 video_frame.timestamp_us() * 1000, &cropped_width,
525 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100526 VideoFrame adapted_frame =
527 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200528 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100529 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200530 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100531 .set_timestamp_ms(99)
532 .set_rotation(kVideoRotation_0)
533 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100534 if (video_frame.has_update_rect()) {
535 adapted_frame.set_update_rect(
536 video_frame.update_rect().ScaleWithFrame(
537 video_frame.width(), video_frame.height(), 0, 0,
538 video_frame.width(), video_frame.height(), out_width,
539 out_height));
540 }
sprangc5d62e22017-04-02 23:53:04 -0700541 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800542 last_width_.emplace(adapted_frame.width());
543 last_height_.emplace(adapted_frame.height());
544 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200545 last_width_ = absl::nullopt;
546 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700547 }
sprangb1ca0732017-02-01 08:38:12 -0800548 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000549 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800550 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800551 last_width_.emplace(video_frame.width());
552 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800553 }
554 }
555
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200556 void OnOutputFormatRequest(int width, int height) {
557 absl::optional<std::pair<int, int>> target_aspect_ratio =
558 std::make_pair(width, height);
559 absl::optional<int> max_pixel_count = width * height;
560 absl::optional<int> max_fps;
561 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
562 max_fps);
563 }
564
sprangb1ca0732017-02-01 08:38:12 -0800565 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
566 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200567 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100568 rtc::VideoSinkWants prev_wants = sink_wants_locked();
569 bool did_adapt =
570 prev_wants.max_pixel_count != wants.max_pixel_count ||
571 prev_wants.target_pixel_count != wants.target_pixel_count ||
572 prev_wants.max_framerate_fps != wants.max_framerate_fps;
573 if (did_adapt) {
574 last_wants_ = prev_wants;
575 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100576 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200577 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800578 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200579
580 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800581 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200582 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
583 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200584 absl::optional<int> last_width_;
585 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800586};
sprangc5d62e22017-04-02 23:53:04 -0700587
Niels Möller213618e2018-07-24 09:29:58 +0200588// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700589class MockableSendStatisticsProxy : public SendStatisticsProxy {
590 public:
591 MockableSendStatisticsProxy(Clock* clock,
592 const VideoSendStream::Config& config,
593 VideoEncoderConfig::ContentType content_type)
594 : SendStatisticsProxy(clock, config, content_type) {}
595
596 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200597 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700598 if (mock_stats_)
599 return *mock_stats_;
600 return SendStatisticsProxy::GetStats();
601 }
602
Niels Möller213618e2018-07-24 09:29:58 +0200603 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200604 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200605 if (mock_stats_)
606 return mock_stats_->input_frame_rate;
607 return SendStatisticsProxy::GetInputFrameRate();
608 }
sprangc5d62e22017-04-02 23:53:04 -0700609 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200610 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700611 mock_stats_.emplace(stats);
612 }
613
614 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200615 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700616 mock_stats_.reset();
617 }
618
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200619 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
620 on_frame_dropped_ = std::move(callback);
621 }
622
sprangc5d62e22017-04-02 23:53:04 -0700623 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200624 void OnFrameDropped(DropReason reason) override {
625 SendStatisticsProxy::OnFrameDropped(reason);
626 if (on_frame_dropped_)
627 on_frame_dropped_(reason);
628 }
629
Markus Handella3765182020-07-08 13:13:32 +0200630 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200631 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200632 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700633};
634
Markus Handellb4e96d42021-11-05 12:00:55 +0100635class SimpleVideoStreamEncoderFactory {
636 public:
637 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
638 public:
639 using VideoStreamEncoder::VideoStreamEncoder;
640 ~AdaptedVideoStreamEncoder() { Stop(); }
641 };
642
643 SimpleVideoStreamEncoderFactory()
644 : time_controller_(Timestamp::Millis(0)),
645 task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
646 stats_proxy_(std::make_unique<MockableSendStatisticsProxy>(
647 time_controller_.GetClock(),
648 VideoSendStream::Config(nullptr),
649 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
650 encoder_settings_(
651 VideoEncoder::Capabilities(/*loss_notification=*/false)),
652 fake_encoder_(time_controller_.GetClock()),
653 encoder_factory_(&fake_encoder_) {
654 encoder_settings_.encoder_factory = &encoder_factory_;
655 }
656
657 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
658 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter) {
659 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
660 time_controller_.GetClock(),
661 /*number_of_cores=*/1,
662 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
663 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handell9a478b52021-11-18 16:07:01 +0100664 std::move(zero_hertz_adapter),
665 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
666 "EncoderQueue", TaskQueueFactory::Priority::NORMAL),
Markus Handellb4e96d42021-11-05 12:00:55 +0100667 VideoStreamEncoder::BitrateAllocationCallbackType::
668 kVideoBitrateAllocation);
669 result->SetSink(&sink_, /*rotation_applied=*/false);
670 return result;
671 }
672
Markus Handell9a478b52021-11-18 16:07:01 +0100673 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
674
Markus Handellb4e96d42021-11-05 12:00:55 +0100675 private:
676 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
677 public:
678 ~NullEncoderSink() override = default;
679 void OnEncoderConfigurationChanged(
680 std::vector<VideoStream> streams,
681 bool is_svc,
682 VideoEncoderConfig::ContentType content_type,
683 int min_transmit_bitrate_bps) override {}
684 void OnBitrateAllocationUpdated(
685 const VideoBitrateAllocation& allocation) override {}
686 void OnVideoLayersAllocationUpdated(
687 VideoLayersAllocation allocation) override {}
688 Result OnEncodedImage(
689 const EncodedImage& encoded_image,
690 const CodecSpecificInfo* codec_specific_info) override {
691 return Result(EncodedImageCallback::Result::OK);
692 }
693 };
694
695 GlobalSimulatedTimeController time_controller_;
696 std::unique_ptr<TaskQueueFactory> task_queue_factory_;
697 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
698 VideoStreamEncoderSettings encoder_settings_;
699 test::FakeEncoder fake_encoder_;
700 test::VideoEncoderProxyFactory encoder_factory_;
701 NullEncoderSink sink_;
702};
703
704class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
705 public:
706 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
707 MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override));
708 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
709};
710
philipel9b058032020-02-10 11:30:00 +0100711class MockEncoderSelector
712 : public VideoEncoderFactory::EncoderSelectorInterface {
713 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200714 MOCK_METHOD(void,
715 OnCurrentEncoder,
716 (const SdpVideoFormat& format),
717 (override));
718 MOCK_METHOD(absl::optional<SdpVideoFormat>,
719 OnAvailableBitrate,
720 (const DataRate& rate),
721 (override));
722 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100723};
724
perkj803d97f2016-11-01 11:45:46 -0700725} // namespace
726
mflodmancc3d4422017-08-03 08:27:51 -0700727class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700728 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200729 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700730
mflodmancc3d4422017-08-03 08:27:51 -0700731 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700732 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700733 codec_width_(320),
734 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200735 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200736 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200737 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700738 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200739 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700740 video_send_config_,
741 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200742 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700743
744 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700745 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700746 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200747 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800748 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200749 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200750 video_send_config_.rtp.payload_name = "FAKE";
751 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700752
Per512ecb32016-09-23 15:52:06 +0200753 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200754 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200755 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
756 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
757 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100758 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700759
Niels Möllerf1338562018-04-26 09:51:47 +0200760 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800761 }
762
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100763 void ConfigureEncoder(
764 VideoEncoderConfig video_encoder_config,
765 VideoStreamEncoder::BitrateAllocationCallbackType
766 allocation_callback_type =
767 VideoStreamEncoder::BitrateAllocationCallbackType::
768 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700769 if (video_stream_encoder_)
770 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100771
772 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
773 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
774 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
775 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
776 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
777 encoder_queue_ptr);
778 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
779 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
780 stats_proxy_.get(), video_send_config_.encoder_settings,
781 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200782 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700783 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700784 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200785 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700786 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200787 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700788 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800789 }
790
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100791 void ResetEncoder(const std::string& payload_name,
792 size_t num_streams,
793 size_t num_temporal_layers,
794 unsigned char num_spatial_layers,
795 bool screenshare,
796 VideoStreamEncoder::BitrateAllocationCallbackType
797 allocation_callback_type =
798 VideoStreamEncoder::BitrateAllocationCallbackType::
799 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200800 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800801
802 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200803 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
804 num_streams, &video_encoder_config);
805 for (auto& layer : video_encoder_config.simulcast_layers) {
806 layer.num_temporal_layers = num_temporal_layers;
807 layer.max_framerate = kDefaultFramerate;
808 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100809 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200810 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700811 video_encoder_config.content_type =
812 screenshare ? VideoEncoderConfig::ContentType::kScreen
813 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700814 if (payload_name == "VP9") {
815 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
816 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200817 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700818 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200819 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
820 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700821 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100822 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700823 }
824
sprang57c2fff2017-01-16 06:24:02 -0800825 VideoFrame CreateFrame(int64_t ntp_time_ms,
826 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200827 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200828 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200829 destruction_event, codec_width_, codec_height_))
830 .set_ntp_time_ms(ntp_time_ms)
831 .set_timestamp_ms(99)
832 .set_rotation(kVideoRotation_0)
833 .build();
perkj26091b12016-09-01 01:17:40 -0700834 }
835
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100836 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
837 rtc::Event* destruction_event,
838 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200839 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200840 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200841 destruction_event, codec_width_, codec_height_))
842 .set_ntp_time_ms(ntp_time_ms)
843 .set_timestamp_ms(99)
844 .set_rotation(kVideoRotation_0)
845 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
846 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100847 }
848
sprang57c2fff2017-01-16 06:24:02 -0800849 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200850 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
851 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200852 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200853 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200854 .set_ntp_time_ms(ntp_time_ms)
855 .set_timestamp_ms(ntp_time_ms)
856 .set_rotation(kVideoRotation_0)
857 .build();
perkj803d97f2016-11-01 11:45:46 -0700858 }
859
Evan Shrubsole895556e2020-10-05 09:15:13 +0200860 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200861 return VideoFrame::Builder()
862 .set_video_frame_buffer(NV12Buffer::Create(width, height))
863 .set_ntp_time_ms(ntp_time_ms)
864 .set_timestamp_ms(ntp_time_ms)
865 .set_rotation(kVideoRotation_0)
866 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200867 }
868
Noah Richards51db4212019-06-12 06:59:12 -0700869 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
870 rtc::Event* destruction_event,
871 int width,
872 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200873 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200874 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200875 destruction_event, width, height))
876 .set_ntp_time_ms(ntp_time_ms)
877 .set_timestamp_ms(99)
878 .set_rotation(kVideoRotation_0)
879 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700880 }
881
Evan Shrubsole895556e2020-10-05 09:15:13 +0200882 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
883 rtc::Event* destruction_event,
884 int width,
885 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200886 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200887 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200888 destruction_event, width, height))
889 .set_ntp_time_ms(ntp_time_ms)
890 .set_timestamp_ms(99)
891 .set_rotation(kVideoRotation_0)
892 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200893 }
894
Noah Richards51db4212019-06-12 06:59:12 -0700895 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
896 rtc::Event* destruction_event) const {
897 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
898 codec_height_);
899 }
900
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100901 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200902 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200903 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100904
905 video_source_.IncomingCapturedFrame(
906 CreateFrame(1, codec_width_, codec_height_));
907 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200908 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100909 }
910
sprang4847ae62017-06-27 07:06:52 -0700911 void WaitForEncodedFrame(int64_t expected_ntp_time) {
912 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200913 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700914 }
915
916 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
917 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200918 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700919 return ok;
920 }
921
922 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
923 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200924 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700925 }
926
927 void ExpectDroppedFrame() {
928 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200929 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700930 }
931
932 bool WaitForFrame(int64_t timeout_ms) {
933 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200934 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700935 return ok;
936 }
937
perkj26091b12016-09-01 01:17:40 -0700938 class TestEncoder : public test::FakeEncoder {
939 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200940 explicit TestEncoder(TimeController* time_controller)
941 : FakeEncoder(time_controller->GetClock()),
942 time_controller_(time_controller) {
943 RTC_DCHECK(time_controller_);
944 }
perkj26091b12016-09-01 01:17:40 -0700945
Erik Språngaed30702018-11-05 12:57:17 +0100946 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200947 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200948 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100949 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100950 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100951 info.scaling_settings = VideoEncoder::ScalingSettings(
952 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100953 }
954 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100955 for (int i = 0; i < kMaxSpatialLayers; ++i) {
956 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100957 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100958 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100959 for (int tid = 0; tid < num_layers; ++tid)
960 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100961 }
962 }
Erik Språngaed30702018-11-05 12:57:17 +0100963 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200964
965 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100966 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200967 info.apply_alignment_to_all_simulcast_layers =
968 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200969 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800970 if (is_qp_trusted_.has_value()) {
971 info.is_qp_trusted = is_qp_trusted_;
972 }
Erik Språngaed30702018-11-05 12:57:17 +0100973 return info;
kthelgason876222f2016-11-29 01:44:11 -0800974 }
975
Erik Språngb7cb7b52019-02-26 15:52:33 +0100976 int32_t RegisterEncodeCompleteCallback(
977 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200978 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100979 encoded_image_callback_ = callback;
980 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
981 }
982
perkjfa10b552016-10-02 23:45:26 -0700983 void ContinueEncode() { continue_encode_event_.Set(); }
984
985 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
986 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200987 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700988 EXPECT_EQ(timestamp_, timestamp);
989 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
990 }
991
kthelgason2fc52542017-03-03 00:24:41 -0800992 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200993 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800994 quality_scaling_ = b;
995 }
kthelgasonad9010c2017-02-14 00:46:51 -0800996
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100997 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200998 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100999 requested_resolution_alignment_ = requested_resolution_alignment;
1000 }
1001
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001002 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1003 MutexLock lock(&local_mutex_);
1004 apply_alignment_to_all_simulcast_layers_ = b;
1005 }
1006
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001007 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001008 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001009 is_hardware_accelerated_ = is_hardware_accelerated;
1010 }
1011
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001012 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1013 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001014 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001015 temporal_layers_supported_[spatial_idx] = supported;
1016 }
1017
Sergey Silkin6456e352019-07-08 17:56:40 +02001018 void SetResolutionBitrateLimits(
1019 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001020 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001021 resolution_bitrate_limits_ = thresholds;
1022 }
1023
sprangfe627f32017-03-29 08:24:59 -07001024 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001025 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001026 force_init_encode_failed_ = force_failure;
1027 }
1028
Niels Möller6bb5ab92019-01-11 11:11:10 +01001029 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001030 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001031 rate_factor_ = rate_factor;
1032 }
1033
Erik Språngd7329ca2019-02-21 21:19:53 +01001034 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001035 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001036 return last_framerate_;
1037 }
1038
Erik Språngd7329ca2019-02-21 21:19:53 +01001039 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001040 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001041 return last_update_rect_;
1042 }
1043
Niels Möller87e2d782019-03-07 10:18:23 +01001044 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001045 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001046 return last_frame_types_;
1047 }
1048
1049 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001050 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001051 keyframe ? VideoFrameType::kVideoFrameKey
1052 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001053 {
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001055 last_frame_types_ = frame_type;
1056 }
Niels Möllerb859b322019-03-07 12:40:01 +01001057 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001058 }
1059
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001060 void InjectEncodedImage(const EncodedImage& image,
1061 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001062 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001063 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001064 }
1065
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001066 void SetEncodedImageData(
1067 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001068 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001069 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001070 }
1071
Erik Språngd7329ca2019-02-21 21:19:53 +01001072 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001073 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001074 expect_null_frame_ = true;
1075 }
1076
Erik Språng5056af02019-09-02 15:53:11 +02001077 absl::optional<VideoEncoder::RateControlParameters>
1078 GetAndResetLastRateControlSettings() {
1079 auto settings = last_rate_control_settings_;
1080 last_rate_control_settings_.reset();
1081 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001082 }
1083
Henrik Boström56db9ff2021-03-24 09:06:45 +01001084 int GetLastInputWidth() const {
1085 MutexLock lock(&local_mutex_);
1086 return last_input_width_;
1087 }
1088
1089 int GetLastInputHeight() const {
1090 MutexLock lock(&local_mutex_);
1091 return last_input_height_;
1092 }
1093
Evan Shrubsole895556e2020-10-05 09:15:13 +02001094 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1095 MutexLock lock(&local_mutex_);
1096 return last_input_pixel_format_;
1097 }
1098
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001099 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001100 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001101 return num_set_rates_;
1102 }
1103
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001104 void SetPreferredPixelFormats(
1105 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1106 pixel_formats) {
1107 MutexLock lock(&local_mutex_);
1108 preferred_pixel_formats_ = std::move(pixel_formats);
1109 }
1110
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001111 void SetIsQpTrusted(absl::optional<bool> trusted) {
1112 MutexLock lock(&local_mutex_);
1113 is_qp_trusted_ = trusted;
1114 }
1115
perkjfa10b552016-10-02 23:45:26 -07001116 private:
perkj26091b12016-09-01 01:17:40 -07001117 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001118 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001119 {
Markus Handella3765182020-07-08 13:13:32 +02001120 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001121 if (expect_null_frame_) {
1122 EXPECT_EQ(input_image.timestamp(), 0u);
1123 EXPECT_EQ(input_image.width(), 1);
1124 last_frame_types_ = *frame_types;
1125 expect_null_frame_ = false;
1126 } else {
1127 EXPECT_GT(input_image.timestamp(), timestamp_);
1128 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1129 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1130 }
perkj26091b12016-09-01 01:17:40 -07001131
1132 timestamp_ = input_image.timestamp();
1133 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001134 last_input_width_ = input_image.width();
1135 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001136 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001138 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001139 }
Niels Möllerb859b322019-03-07 12:40:01 +01001140 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001141 return result;
1142 }
1143
Niels Möller08ae7ce2020-09-23 15:58:12 +02001144 CodecSpecificInfo EncodeHook(
1145 EncodedImage& encoded_image,
1146 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001147 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001148 {
1149 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001150 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001151 }
1152 MutexLock lock(&local_mutex_);
1153 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001154 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001155 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001156 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001157 }
1158
sprangfe627f32017-03-29 08:24:59 -07001159 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001160 const Settings& settings) override {
1161 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001162
Markus Handella3765182020-07-08 13:13:32 +02001163 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001164 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001165
Erik Språng82fad3d2018-03-21 09:57:23 +01001166 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001167 // Simulate setting up temporal layers, in order to validate the life
1168 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001169 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001170 frame_buffer_controller_ =
1171 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001172 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001173 if (force_init_encode_failed_) {
1174 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001175 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001176 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001177
Erik Språngb7cb7b52019-02-26 15:52:33 +01001178 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001179 return res;
1180 }
1181
Erik Språngb7cb7b52019-02-26 15:52:33 +01001182 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001183 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001184 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1185 initialized_ = EncoderState::kUninitialized;
1186 return FakeEncoder::Release();
1187 }
1188
Erik Språng16cb8f52019-04-12 13:59:09 +02001189 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001190 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001191 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001192 VideoBitrateAllocation adjusted_rate_allocation;
1193 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1194 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001195 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001196 adjusted_rate_allocation.SetBitrate(
1197 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001198 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001199 rate_factor_));
1200 }
1201 }
1202 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001203 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001204 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001205 RateControlParameters adjusted_paramters = parameters;
1206 adjusted_paramters.bitrate = adjusted_rate_allocation;
1207 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001208 }
1209
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001210 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001211 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001212 enum class EncoderState {
1213 kUninitialized,
1214 kInitializationFailed,
1215 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001216 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001217 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001218 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1219 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1220 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1221 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1222 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1223 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001224 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1225 false;
Markus Handella3765182020-07-08 13:13:32 +02001226 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001227 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1228 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001229 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001230 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001231 absl::optional<bool>
1232 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001233 local_mutex_);
1234 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1235 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1236 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001237 absl::optional<VideoEncoder::RateControlParameters>
1238 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001239 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1240 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001241 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001242 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001243 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1244 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001245 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001246 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001247 RTC_GUARDED_BY(local_mutex_);
1248 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001249 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1250 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001251 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1252 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001253 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001254 };
1255
mflodmancc3d4422017-08-03 08:27:51 -07001256 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001257 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001258 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1259 : time_controller_(time_controller), test_encoder_(test_encoder) {
1260 RTC_DCHECK(time_controller_);
1261 }
perkj26091b12016-09-01 01:17:40 -07001262
perkj26091b12016-09-01 01:17:40 -07001263 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001264 EXPECT_TRUE(
1265 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1266 }
1267
1268 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1269 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001270 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001271 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001272 return false;
perkj26091b12016-09-01 01:17:40 -07001273 {
Markus Handella3765182020-07-08 13:13:32 +02001274 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001275 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001276 }
1277 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001278 return true;
perkj26091b12016-09-01 01:17:40 -07001279 }
1280
sprangb1ca0732017-02-01 08:38:12 -08001281 void WaitForEncodedFrame(uint32_t expected_width,
1282 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001283 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001284 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001285 }
1286
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001287 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001288 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001289 uint32_t width = 0;
1290 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001291 {
Markus Handella3765182020-07-08 13:13:32 +02001292 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001293 width = last_width_;
1294 height = last_height_;
1295 }
1296 EXPECT_EQ(expected_height, height);
1297 EXPECT_EQ(expected_width, width);
1298 }
1299
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001300 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1301 VideoRotation rotation;
1302 {
Markus Handella3765182020-07-08 13:13:32 +02001303 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001304 rotation = last_rotation_;
1305 }
1306 EXPECT_EQ(expected_rotation, rotation);
1307 }
1308
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001309 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001310
sprangc5d62e22017-04-02 23:53:04 -07001311 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001312 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001313 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001314 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001315 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001316 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001317 }
1318
perkj26091b12016-09-01 01:17:40 -07001319 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001320 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001321 expect_frames_ = false;
1322 }
1323
asaperssonfab67072017-04-04 05:51:49 -07001324 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001325 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001326 return number_of_reconfigurations_;
1327 }
1328
asaperssonfab67072017-04-04 05:51:49 -07001329 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001330 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001331 return min_transmit_bitrate_bps_;
1332 }
1333
Erik Språngd7329ca2019-02-21 21:19:53 +01001334 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001335 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001336 num_expected_layers_ = num_layers;
1337 }
1338
Erik Språngb7cb7b52019-02-26 15:52:33 +01001339 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001340 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001341 return last_capture_time_ms_;
1342 }
1343
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001344 const EncodedImage& GetLastEncodedImage() {
1345 MutexLock lock(&mutex_);
1346 return last_encoded_image_;
1347 }
1348
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001349 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001350 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001351 return std::move(last_encoded_image_data_);
1352 }
1353
Per Kjellanderdcef6412020-10-07 15:09:05 +02001354 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1355 MutexLock lock(&mutex_);
1356 return last_bitrate_allocation_;
1357 }
1358
1359 int number_of_bitrate_allocations() const {
1360 MutexLock lock(&mutex_);
1361 return number_of_bitrate_allocations_;
1362 }
1363
Per Kjellandera9434842020-10-15 17:53:22 +02001364 VideoLayersAllocation GetLastVideoLayersAllocation() {
1365 MutexLock lock(&mutex_);
1366 return last_layers_allocation_;
1367 }
1368
1369 int number_of_layers_allocations() const {
1370 MutexLock lock(&mutex_);
1371 return number_of_layers_allocations_;
1372 }
1373
perkj26091b12016-09-01 01:17:40 -07001374 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001375 Result OnEncodedImage(
1376 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001377 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001378 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001379 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001380 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001381 last_encoded_image_data_ = std::vector<uint8_t>(
1382 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001383 uint32_t timestamp = encoded_image.Timestamp();
1384 if (last_timestamp_ != timestamp) {
1385 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001386 last_width_ = encoded_image._encodedWidth;
1387 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001388 } else {
1389 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001390 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1391 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001392 }
1393 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001394 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001395 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001396 if (num_received_layers_ == num_expected_layers_) {
1397 encoded_frame_event_.Set();
1398 }
sprangb1ca0732017-02-01 08:38:12 -08001399 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001400 }
1401
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001402 void OnEncoderConfigurationChanged(
1403 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001404 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001405 VideoEncoderConfig::ContentType content_type,
1406 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001407 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001408 ++number_of_reconfigurations_;
1409 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1410 }
1411
Per Kjellanderdcef6412020-10-07 15:09:05 +02001412 void OnBitrateAllocationUpdated(
1413 const VideoBitrateAllocation& allocation) override {
1414 MutexLock lock(&mutex_);
1415 ++number_of_bitrate_allocations_;
1416 last_bitrate_allocation_ = allocation;
1417 }
1418
Per Kjellandera9434842020-10-15 17:53:22 +02001419 void OnVideoLayersAllocationUpdated(
1420 VideoLayersAllocation allocation) override {
1421 MutexLock lock(&mutex_);
1422 ++number_of_layers_allocations_;
1423 last_layers_allocation_ = allocation;
1424 rtc::StringBuilder log;
1425 for (const auto& layer : allocation.active_spatial_layers) {
1426 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1427 << "[";
1428 for (const auto target_bitrate :
1429 layer.target_bitrate_per_temporal_layer) {
1430 log << target_bitrate.kbps() << ",";
1431 }
1432 log << "]";
1433 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001434 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001435 }
1436
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001437 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001438 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001439 TestEncoder* test_encoder_;
1440 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001441 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001442 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001443 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001444 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001445 uint32_t last_height_ = 0;
1446 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001447 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001448 size_t num_expected_layers_ = 1;
1449 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001450 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001451 int number_of_reconfigurations_ = 0;
1452 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001453 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1454 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001455 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1456 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001457 };
1458
Sergey Silkin5ee69672019-07-02 14:18:34 +02001459 class VideoBitrateAllocatorProxyFactory
1460 : public VideoBitrateAllocatorFactory {
1461 public:
1462 VideoBitrateAllocatorProxyFactory()
1463 : bitrate_allocator_factory_(
1464 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1465
1466 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1467 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001468 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001469 codec_config_ = codec;
1470 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1471 }
1472
1473 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001474 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001475 return codec_config_;
1476 }
1477
1478 private:
1479 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1480
Markus Handella3765182020-07-08 13:13:32 +02001481 mutable Mutex mutex_;
1482 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001483 };
1484
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001485 Clock* clock() { return time_controller_.GetClock(); }
1486 void AdvanceTime(TimeDelta duration) {
1487 time_controller_.AdvanceTime(duration);
1488 }
1489
1490 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1491
1492 protected:
1493 virtual TaskQueueFactory* GetTaskQueueFactory() {
1494 return time_controller_.GetTaskQueueFactory();
1495 }
1496
1497 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001498 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001499 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001500 int codec_width_;
1501 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001502 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001503 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001504 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001505 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001506 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001507 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001508 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001509 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001510};
1511
mflodmancc3d4422017-08-03 08:27:51 -07001512TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001514 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001515 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001516 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001517 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001518 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001519 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001520}
1521
mflodmancc3d4422017-08-03 08:27:51 -07001522TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001523 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001524 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001525 // The encoder will cache up to one frame for a short duration. Adding two
1526 // frames means that the first frame will be dropped and the second frame will
1527 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001528 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001529 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001530 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001531 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001532 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001533
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001535 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001536
Sebastian Janssona3177052018-04-10 13:05:49 +02001537 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001538 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001539 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1540
1541 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001542 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001543}
1544
mflodmancc3d4422017-08-03 08:27:51 -07001545TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001547 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001548 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001549 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001550
Henrik Boström381d1092020-05-12 18:49:07 +02001551 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001552 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1553
Sebastian Janssona3177052018-04-10 13:05:49 +02001554 // The encoder will cache up to one frame for a short duration. Adding two
1555 // frames means that the first frame will be dropped and the second frame will
1556 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001557 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001558 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001559
Henrik Boström381d1092020-05-12 18:49:07 +02001560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001561 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001562 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001563 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1564 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001565 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001566}
1567
mflodmancc3d4422017-08-03 08:27:51 -07001568TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001570 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001571 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001572 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001573
1574 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001575 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001576
perkja49cbd32016-09-16 07:53:41 -07001577 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001578 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001579 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001580}
1581
mflodmancc3d4422017-08-03 08:27:51 -07001582TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001583 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001584 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001585
perkja49cbd32016-09-16 07:53:41 -07001586 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001587 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001588
mflodmancc3d4422017-08-03 08:27:51 -07001589 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001590 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001591 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001592 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1593 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001594}
1595
Markus Handell9a478b52021-11-18 16:07:01 +01001596TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1597 test::FrameForwarder source;
1598 video_stream_encoder_->SetSource(&source,
1599 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001600 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001601 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001602
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001603 int dropped_count = 0;
1604 stats_proxy_->SetDroppedFrameCallback(
1605 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1606 ++dropped_count;
1607 });
1608
Markus Handell9a478b52021-11-18 16:07:01 +01001609 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1610 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1611 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001612 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001613 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001614}
1615
Henrik Boström56db9ff2021-03-24 09:06:45 +01001616TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001617 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001618 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001619
1620 rtc::Event frame_destroyed_event;
1621 video_source_.IncomingCapturedFrame(
1622 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001623 WaitForEncodedFrame(1);
1624 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1625 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001626 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1627 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001628 video_stream_encoder_->Stop();
1629}
1630
Henrik Boström56db9ff2021-03-24 09:06:45 +01001631TEST_F(VideoStreamEncoderTest,
1632 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001633 // Use the cropping factory.
1634 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001635 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001636 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1637 kMaxPayloadLength);
1638 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1639
1640 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001642 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001643 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1644 WaitForEncodedFrame(1);
1645 // The encoder will have been configured once.
1646 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001647 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1648 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001649
1650 // Now send in a fake frame that needs to be cropped as the width/height
1651 // aren't divisible by 4 (see CreateEncoderStreams above).
1652 rtc::Event frame_destroyed_event;
1653 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1654 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001655 WaitForEncodedFrame(2);
1656 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1657 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001658 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1659 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001660 video_stream_encoder_->Stop();
1661}
1662
Evan Shrubsole895556e2020-10-05 09:15:13 +02001663TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001665 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001666
1667 video_source_.IncomingCapturedFrame(
1668 CreateNV12Frame(1, codec_width_, codec_height_));
1669 WaitForEncodedFrame(1);
1670 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1671 fake_encoder_.GetLastInputPixelFormat());
1672 video_stream_encoder_->Stop();
1673}
1674
Henrik Boström56db9ff2021-03-24 09:06:45 +01001675TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001678
1679 fake_encoder_.SetPreferredPixelFormats({});
1680
1681 rtc::Event frame_destroyed_event;
1682 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1683 1, &frame_destroyed_event, codec_width_, codec_height_));
1684 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001685 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001686 fake_encoder_.GetLastInputPixelFormat());
1687 video_stream_encoder_->Stop();
1688}
1689
Henrik Boström56db9ff2021-03-24 09:06:45 +01001690TEST_F(VideoStreamEncoderTest,
1691 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001692 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001693 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001694
1695 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1696
1697 rtc::Event frame_destroyed_event;
1698 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1699 1, &frame_destroyed_event, codec_width_, codec_height_));
1700 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001701 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001702 fake_encoder_.GetLastInputPixelFormat());
1703 video_stream_encoder_->Stop();
1704}
1705
Henrik Boström56db9ff2021-03-24 09:06:45 +01001706TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001709
1710 // Fake NV12 native frame does not allow mapping to I444.
1711 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1712
1713 rtc::Event frame_destroyed_event;
1714 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1715 1, &frame_destroyed_event, codec_width_, codec_height_));
1716 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001717 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001718 fake_encoder_.GetLastInputPixelFormat());
1719 video_stream_encoder_->Stop();
1720}
1721
Henrik Boström56db9ff2021-03-24 09:06:45 +01001722TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001725
1726 rtc::Event frame_destroyed_event;
1727 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1728 1, &frame_destroyed_event, codec_width_, codec_height_));
1729 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001730 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001731 fake_encoder_.GetLastInputPixelFormat());
1732 video_stream_encoder_->Stop();
1733}
1734
Ying Wang9b881ab2020-02-07 14:29:32 +01001735TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001736 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001737 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001738 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1739 WaitForEncodedFrame(1);
1740
Henrik Boström381d1092020-05-12 18:49:07 +02001741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001742 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001743 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1744 // frames. Adding two frames means that the first frame will be dropped and
1745 // the second frame will be sent to the encoder.
1746 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1747 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1748 WaitForEncodedFrame(3);
1749 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1750 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1751 WaitForEncodedFrame(5);
1752 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1753 video_stream_encoder_->Stop();
1754}
1755
mflodmancc3d4422017-08-03 08:27:51 -07001756TEST_F(VideoStreamEncoderTest,
1757 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001759 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001760 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001761
1762 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001763 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001764 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001765 // The encoder will have been configured once when the first frame is
1766 // received.
1767 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001768
1769 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001770 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001771 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001772 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001773 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001774
1775 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001776 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001777 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001778 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001779 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001780
mflodmancc3d4422017-08-03 08:27:51 -07001781 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001782}
1783
mflodmancc3d4422017-08-03 08:27:51 -07001784TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001786 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001787
1788 // Capture a frame and wait for it to synchronize with the encoder thread.
1789 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001790 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001791 // The encoder will have been configured once.
1792 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001793 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1794 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001795
1796 codec_width_ *= 2;
1797 codec_height_ *= 2;
1798 // Capture a frame with a higher resolution and wait for it to synchronize
1799 // with the encoder thread.
1800 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001801 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001802 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1803 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001804 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001805
mflodmancc3d4422017-08-03 08:27:51 -07001806 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001807}
1808
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001809TEST_F(VideoStreamEncoderTest,
1810 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001812 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001813
1814 // Capture a frame and wait for it to synchronize with the encoder thread.
1815 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1816 WaitForEncodedFrame(1);
1817
1818 VideoEncoderConfig video_encoder_config;
1819 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1820 // Changing the max payload data length recreates encoder.
1821 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1822 kMaxPayloadLength / 2);
1823
1824 // Capture a frame and wait for it to synchronize with the encoder thread.
1825 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1826 WaitForEncodedFrame(2);
1827 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1828
1829 video_stream_encoder_->Stop();
1830}
1831
Sergey Silkin5ee69672019-07-02 14:18:34 +02001832TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001835
1836 VideoEncoderConfig video_encoder_config;
1837 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001838 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1839 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001840 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1841 kMaxPayloadLength);
1842
1843 // Capture a frame and wait for it to synchronize with the encoder thread.
1844 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1845 WaitForEncodedFrame(1);
1846 // The encoder will have been configured once when the first frame is
1847 // received.
1848 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001849 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001850 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001851 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001852 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1853
Sergey Silkin6456e352019-07-08 17:56:40 +02001854 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1855 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001856 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1857 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001858 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1859 kMaxPayloadLength);
1860
1861 // Capture a frame and wait for it to synchronize with the encoder thread.
1862 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1863 WaitForEncodedFrame(2);
1864 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1865 // Bitrate limits have changed - rate allocator should be reconfigured,
1866 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001867 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001868 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001869 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001870 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001871 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001872
1873 video_stream_encoder_->Stop();
1874}
1875
Sergey Silkin6456e352019-07-08 17:56:40 +02001876TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001877 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001878 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001879 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001880
Sergey Silkincd02eba2020-01-20 14:48:40 +01001881 const uint32_t kMinEncBitrateKbps = 100;
1882 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001884 /*frame_size_pixels=*/codec_width_ * codec_height_,
1885 /*min_start_bitrate_bps=*/0,
1886 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1887 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001888 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1889
Sergey Silkincd02eba2020-01-20 14:48:40 +01001890 VideoEncoderConfig video_encoder_config;
1891 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1892 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1893 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1894 (kMinEncBitrateKbps + 1) * 1000;
1895 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1896 kMaxPayloadLength);
1897
1898 // When both encoder and app provide bitrate limits, the intersection of
1899 // provided sets should be used.
1900 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1901 WaitForEncodedFrame(1);
1902 EXPECT_EQ(kMaxEncBitrateKbps,
1903 bitrate_allocator_factory_.codec_config().maxBitrate);
1904 EXPECT_EQ(kMinEncBitrateKbps + 1,
1905 bitrate_allocator_factory_.codec_config().minBitrate);
1906
1907 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1908 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1909 (kMinEncBitrateKbps - 1) * 1000;
1910 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1911 kMaxPayloadLength);
1912 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001913 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001914 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001915 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001916 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001917 bitrate_allocator_factory_.codec_config().minBitrate);
1918
Sergey Silkincd02eba2020-01-20 14:48:40 +01001919 video_stream_encoder_->Stop();
1920}
1921
1922TEST_F(VideoStreamEncoderTest,
1923 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001926
1927 const uint32_t kMinAppBitrateKbps = 100;
1928 const uint32_t kMaxAppBitrateKbps = 200;
1929 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1930 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1931 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1932 /*frame_size_pixels=*/codec_width_ * codec_height_,
1933 /*min_start_bitrate_bps=*/0,
1934 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1935 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1936 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1937
1938 VideoEncoderConfig video_encoder_config;
1939 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1940 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1941 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1942 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001943 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1944 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001945
Sergey Silkincd02eba2020-01-20 14:48:40 +01001946 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1947 WaitForEncodedFrame(1);
1948 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001950 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001951 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001952
1953 video_stream_encoder_->Stop();
1954}
1955
1956TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001957 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001959 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001960
1961 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001963 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001965 fake_encoder_.SetResolutionBitrateLimits(
1966 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1967
1968 VideoEncoderConfig video_encoder_config;
1969 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1970 video_encoder_config.max_bitrate_bps = 0;
1971 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1972 kMaxPayloadLength);
1973
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001974 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001975 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1976 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001977 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1978 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001979 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1980 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1981
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001982 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001983 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1984 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001985 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1986 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001987 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1988 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1989
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001991 // encoder for 360p should be used.
1992 video_source_.IncomingCapturedFrame(
1993 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1994 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001997 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1998 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1999
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002000 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002001 // ignored.
2002 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2003 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002006 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2007 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2009 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002010 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2011 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2012
2013 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2014 // for 270p should be used.
2015 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2016 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2018 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002019 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2020 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2021
2022 video_stream_encoder_->Stop();
2023}
2024
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002027 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002028
2029 VideoEncoderConfig video_encoder_config;
2030 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2031 video_encoder_config.max_bitrate_bps = 0;
2032 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2033 kMaxPayloadLength);
2034
2035 // Encode 720p frame to get the default encoder target bitrate.
2036 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2037 WaitForEncodedFrame(1);
2038 const uint32_t kDefaultTargetBitrateFor720pKbps =
2039 bitrate_allocator_factory_.codec_config()
2040 .simulcastStream[0]
2041 .targetBitrate;
2042
2043 // Set the max recommended encoder bitrate to something lower than the default
2044 // target bitrate.
2045 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2046 1280 * 720, 10 * 1000, 10 * 1000,
2047 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2048 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2049
2050 // Change resolution to trigger encoder reinitialization.
2051 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2052 WaitForEncodedFrame(2);
2053 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2054 WaitForEncodedFrame(3);
2055
2056 // Ensure the target bitrate is capped by the max bitrate.
2057 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2058 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2059 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2060 .simulcastStream[0]
2061 .targetBitrate *
2062 1000,
2063 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2064
2065 video_stream_encoder_->Stop();
2066}
2067
Åsa Perssona7e34d32021-01-20 15:36:13 +01002068TEST_F(VideoStreamEncoderTest,
2069 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2070 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2071 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2072 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2073 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2074 fake_encoder_.SetResolutionBitrateLimits(
2075 {kEncoderLimits270p, kEncoderLimits360p});
2076
2077 // Two streams, highest stream active.
2078 VideoEncoderConfig config;
2079 const int kNumStreams = 2;
2080 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2081 config.max_bitrate_bps = 0;
2082 config.simulcast_layers[0].active = false;
2083 config.simulcast_layers[1].active = true;
2084 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002085 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002086 "VP8", /*max qp*/ 56, /*screencast*/ false,
2087 /*screenshare enabled*/ false);
2088 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2089
2090 // The encoder bitrate limits for 270p should be used.
2091 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2092 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002093 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002094 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002095 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002096 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002097 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002098
2099 // The encoder bitrate limits for 360p should be used.
2100 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2101 EXPECT_FALSE(WaitForFrame(1000));
2102 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002103 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002104 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002105 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002106
2107 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2108 video_source_.IncomingCapturedFrame(
2109 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2110 EXPECT_FALSE(WaitForFrame(1000));
2111 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002112 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002114 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002115
2116 // Resolution higher than 360p. Encoder limits should be ignored.
2117 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2118 EXPECT_FALSE(WaitForFrame(1000));
2119 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002120 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002121 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002122 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002123 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002124 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002125 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002126 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002127
2128 // Resolution lower than 270p. The encoder limits for 270p should be used.
2129 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2130 EXPECT_FALSE(WaitForFrame(1000));
2131 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002132 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002133 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002134 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002135
2136 video_stream_encoder_->Stop();
2137}
2138
2139TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002140 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2141 // Two streams, highest stream active.
2142 VideoEncoderConfig config;
2143 const int kNumStreams = 2;
2144 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2145 config.max_bitrate_bps = 0;
2146 config.simulcast_layers[0].active = false;
2147 config.simulcast_layers[1].active = true;
2148 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002149 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002150 "VP8", /*max qp*/ 56, /*screencast*/ false,
2151 /*screenshare enabled*/ false);
2152 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2153
2154 // Default bitrate limits for 270p should be used.
2155 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2156 kDefaultLimits270p =
2157 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002158 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002159 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2160 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002161 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002162 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002163 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002164 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002165 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002166
2167 // Default bitrate limits for 360p should be used.
2168 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2169 kDefaultLimits360p =
2170 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002171 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002172 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2173 EXPECT_FALSE(WaitForFrame(1000));
2174 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002175 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002176 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002177 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002178
2179 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2180 video_source_.IncomingCapturedFrame(
2181 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2182 EXPECT_FALSE(WaitForFrame(1000));
2183 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002184 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002185 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002186 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002187
2188 // Default bitrate limits for 540p should be used.
2189 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2190 kDefaultLimits540p =
2191 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002192 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002193 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2194 EXPECT_FALSE(WaitForFrame(1000));
2195 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002196 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002197 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002198 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002199
2200 video_stream_encoder_->Stop();
2201}
2202
2203TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2205 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2206 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2207 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2208 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2209 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2210 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2211 fake_encoder_.SetResolutionBitrateLimits(
2212 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2213
2214 // Three streams, middle stream active.
2215 VideoEncoderConfig config;
2216 const int kNumStreams = 3;
2217 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2218 config.simulcast_layers[0].active = false;
2219 config.simulcast_layers[1].active = true;
2220 config.simulcast_layers[2].active = false;
2221 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002222 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 "VP8", /*max qp*/ 56, /*screencast*/ false,
2224 /*screenshare enabled*/ false);
2225 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2226
2227 // The encoder bitrate limits for 360p should be used.
2228 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2229 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235
2236 // The encoder bitrate limits for 270p should be used.
2237 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2238 EXPECT_FALSE(WaitForFrame(1000));
2239 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002240 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243
2244 video_stream_encoder_->Stop();
2245}
2246
2247TEST_F(VideoStreamEncoderTest,
2248 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2249 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2250 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2251 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2252 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2253 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2254 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2255 fake_encoder_.SetResolutionBitrateLimits(
2256 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2257
2258 // Three streams, lowest stream active.
2259 VideoEncoderConfig config;
2260 const int kNumStreams = 3;
2261 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2262 config.simulcast_layers[0].active = true;
2263 config.simulcast_layers[1].active = false;
2264 config.simulcast_layers[2].active = false;
2265 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002266 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002267 "VP8", /*max qp*/ 56, /*screencast*/ false,
2268 /*screenshare enabled*/ false);
2269 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2270
2271 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2272 // on lowest stream, limits for 270p should not be used
2273 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2274 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002276 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002277 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002278 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002279 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002280
2281 video_stream_encoder_->Stop();
2282}
2283
2284TEST_F(VideoStreamEncoderTest,
2285 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2286 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2287 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2288 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2289 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2290 fake_encoder_.SetResolutionBitrateLimits(
2291 {kEncoderLimits270p, kEncoderLimits360p});
2292 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2293
2294 // Two streams, highest stream active.
2295 VideoEncoderConfig config;
2296 const int kNumStreams = 2;
2297 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2298 config.simulcast_layers[0].active = false;
2299 config.simulcast_layers[1].active = true;
2300 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2301 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002302 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002303 "VP8", /*max qp*/ 56, /*screencast*/ false,
2304 /*screenshare enabled*/ false);
2305 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2306
2307 // The encoder bitrate limits for 270p should be used.
2308 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2309 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002310 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002311 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002312 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002314 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002315
2316 // The max configured bitrate is less than the encoder limit for 360p.
2317 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2318 EXPECT_FALSE(WaitForFrame(1000));
2319 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002320 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002321 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002322 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002323
2324 video_stream_encoder_->Stop();
2325}
2326
mflodmancc3d4422017-08-03 08:27:51 -07002327TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002328 EXPECT_TRUE(video_source_.has_sinks());
2329 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002330 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002331 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002332 EXPECT_FALSE(video_source_.has_sinks());
2333 EXPECT_TRUE(new_video_source.has_sinks());
2334
mflodmancc3d4422017-08-03 08:27:51 -07002335 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002336}
2337
mflodmancc3d4422017-08-03 08:27:51 -07002338TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002339 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002340 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002341 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002342 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002343}
2344
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002345class ResolutionAlignmentTest
2346 : public VideoStreamEncoderTest,
2347 public ::testing::WithParamInterface<
2348 ::testing::tuple<int, std::vector<double>>> {
2349 public:
2350 ResolutionAlignmentTest()
2351 : requested_alignment_(::testing::get<0>(GetParam())),
2352 scale_factors_(::testing::get<1>(GetParam())) {}
2353
2354 protected:
2355 const int requested_alignment_;
2356 const std::vector<double> scale_factors_;
2357};
2358
2359INSTANTIATE_TEST_SUITE_P(
2360 AlignmentAndScaleFactors,
2361 ResolutionAlignmentTest,
2362 ::testing::Combine(
2363 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2364 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2365 std::vector<double>{-1.0, -1.0},
2366 std::vector<double>{-1.0, -1.0, -1.0},
2367 std::vector<double>{4.0, 2.0, 1.0},
2368 std::vector<double>{9999.0, -1.0, 1.0},
2369 std::vector<double>{3.99, 2.01, 1.0},
2370 std::vector<double>{4.9, 1.7, 1.25},
2371 std::vector<double>{10.0, 4.0, 3.0},
2372 std::vector<double>{1.75, 3.5},
2373 std::vector<double>{1.5, 2.5},
2374 std::vector<double>{1.3, 1.0})));
2375
2376TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2377 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002378 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002379 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2380 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2381
2382 // Fill config with the scaling factor by which to reduce encoding size.
2383 const int num_streams = scale_factors_.size();
2384 VideoEncoderConfig config;
2385 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2386 for (int i = 0; i < num_streams; ++i) {
2387 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2388 }
2389 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002390 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002391 "VP8", /*max qp*/ 56, /*screencast*/ false,
2392 /*screenshare enabled*/ false);
2393 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2394
Henrik Boström381d1092020-05-12 18:49:07 +02002395 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002396 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2397 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002398 // Wait for all layers before triggering event.
2399 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002400
2401 // On the 1st frame, we should have initialized the encoder and
2402 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002403 int64_t timestamp_ms = kFrameIntervalMs;
2404 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2405 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002406 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002407
2408 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2409 // (It's up the to the encoder to potentially drop the previous frame,
2410 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002411 timestamp_ms += kFrameIntervalMs;
2412 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2413 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002414 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002415
Asa Persson606d3cb2021-10-04 10:07:11 +02002416 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002417 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2418 // Frame size should be a multiple of the requested alignment.
2419 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2420 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2421 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2422 // Aspect ratio should match.
2423 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2424 codec.height * codec.simulcastStream[i].width);
2425 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002426
2427 video_stream_encoder_->Stop();
2428}
2429
Jonathan Yubc771b72017-12-08 17:04:29 -08002430TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2431 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002432 const int kWidth = 1280;
2433 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002434
2435 // We rely on the automatic resolution adaptation, but we handle framerate
2436 // adaptation manually by mocking the stats proxy.
2437 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002438
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002439 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002441 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002442 video_stream_encoder_->SetSource(&video_source_,
2443 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002444 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002445 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002446 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002447 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2448
Jonathan Yubc771b72017-12-08 17:04:29 -08002449 // Adapt down as far as possible.
2450 rtc::VideoSinkWants last_wants;
2451 int64_t t = 1;
2452 int loop_count = 0;
2453 do {
2454 ++loop_count;
2455 last_wants = video_source_.sink_wants();
2456
2457 // Simulate the framerate we've been asked to adapt to.
2458 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2459 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2460 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2461 mock_stats.input_frame_rate = fps;
2462 stats_proxy_->SetMockStats(mock_stats);
2463
2464 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2465 sink_.WaitForEncodedFrame(t);
2466 t += frame_interval_ms;
2467
mflodmancc3d4422017-08-03 08:27:51 -07002468 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002469 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002470 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002471 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2472 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002473 } while (video_source_.sink_wants().max_pixel_count <
2474 last_wants.max_pixel_count ||
2475 video_source_.sink_wants().max_framerate_fps <
2476 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002477
Jonathan Yubc771b72017-12-08 17:04:29 -08002478 // Verify that we've adapted all the way down.
2479 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002481 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2482 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002483 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002484 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2485 *video_source_.last_sent_height());
2486 EXPECT_EQ(kMinBalancedFramerateFps,
2487 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002488
Jonathan Yubc771b72017-12-08 17:04:29 -08002489 // Adapt back up the same number of times we adapted down.
2490 for (int i = 0; i < loop_count - 1; ++i) {
2491 last_wants = video_source_.sink_wants();
2492
2493 // Simulate the framerate we've been asked to adapt to.
2494 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2495 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2496 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2497 mock_stats.input_frame_rate = fps;
2498 stats_proxy_->SetMockStats(mock_stats);
2499
2500 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2501 sink_.WaitForEncodedFrame(t);
2502 t += frame_interval_ms;
2503
Henrik Boström91aa7322020-04-28 12:24:33 +02002504 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002505 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002506 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002507 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2508 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002509 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2510 last_wants.max_pixel_count ||
2511 video_source_.sink_wants().max_framerate_fps >
2512 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002513 }
2514
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002515 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002516 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002517 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002518 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2519 EXPECT_EQ((loop_count - 1) * 2,
2520 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002521
mflodmancc3d4422017-08-03 08:27:51 -07002522 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002523}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002524
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002525TEST_F(VideoStreamEncoderTest,
2526 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002527 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2528 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002529 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002530
2531 const int kFrameWidth = 1280;
2532 const int kFrameHeight = 720;
2533
2534 int64_t ntp_time = kFrameIntervalMs;
2535
2536 // Force an input frame rate to be available, or the adaptation call won't
2537 // know what framerate to adapt form.
2538 const int kInputFps = 30;
2539 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2540 stats.input_frame_rate = kInputFps;
2541 stats_proxy_->SetMockStats(stats);
2542
2543 video_source_.set_adaptation_enabled(true);
2544 video_stream_encoder_->SetSource(
2545 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002546 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002547 video_source_.IncomingCapturedFrame(
2548 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2549 sink_.WaitForEncodedFrame(ntp_time);
2550 ntp_time += kFrameIntervalMs;
2551
2552 // Trigger CPU overuse.
2553 video_stream_encoder_->TriggerCpuOveruse();
2554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2560 EXPECT_EQ(std::numeric_limits<int>::max(),
2561 video_source_.sink_wants().max_pixel_count);
2562 // Some framerate constraint should be set.
2563 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2564 EXPECT_LT(restricted_fps, kInputFps);
2565 video_source_.IncomingCapturedFrame(
2566 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2567 sink_.WaitForEncodedFrame(ntp_time);
2568 ntp_time += 100;
2569
Henrik Boström2671dac2020-05-19 16:29:09 +02002570 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002571 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2572 // Give the encoder queue time to process the change in degradation preference
2573 // by waiting for an encoded frame.
2574 video_source_.IncomingCapturedFrame(
2575 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2576 sink_.WaitForEncodedFrame(ntp_time);
2577 ntp_time += kFrameIntervalMs;
2578
2579 video_stream_encoder_->TriggerQualityLow();
2580 video_source_.IncomingCapturedFrame(
2581 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2582 sink_.WaitForEncodedFrame(ntp_time);
2583 ntp_time += kFrameIntervalMs;
2584
2585 // Some resolution constraint should be set.
2586 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2587 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2588 kFrameWidth * kFrameHeight);
2589 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2590
2591 int pixel_count = video_source_.sink_wants().max_pixel_count;
2592 // Triggering a CPU underuse should not change the sink wants since it has
2593 // not been overused for resolution since we changed degradation preference.
2594 video_stream_encoder_->TriggerCpuUnderuse();
2595 video_source_.IncomingCapturedFrame(
2596 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2597 sink_.WaitForEncodedFrame(ntp_time);
2598 ntp_time += kFrameIntervalMs;
2599 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2600 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2601
Evan Shrubsole64469032020-06-11 10:45:29 +02002602 // Change the degradation preference back. CPU underuse should not adapt since
2603 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002604 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002605 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2606 video_source_.IncomingCapturedFrame(
2607 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2608 sink_.WaitForEncodedFrame(ntp_time);
2609 ntp_time += 100;
2610 // Resolution adaptations is gone after changing degradation preference.
2611 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2612 EXPECT_EQ(std::numeric_limits<int>::max(),
2613 video_source_.sink_wants().max_pixel_count);
2614 // The fps adaptation from above is now back.
2615 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2616
2617 // Trigger CPU underuse.
2618 video_stream_encoder_->TriggerCpuUnderuse();
2619 video_source_.IncomingCapturedFrame(
2620 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2621 sink_.WaitForEncodedFrame(ntp_time);
2622 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002623 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2624
2625 // Trigger QP underuse, fps should return to normal.
2626 video_stream_encoder_->TriggerQualityHigh();
2627 video_source_.IncomingCapturedFrame(
2628 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2629 sink_.WaitForEncodedFrame(ntp_time);
2630 ntp_time += kFrameIntervalMs;
2631 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002632
2633 video_stream_encoder_->Stop();
2634}
2635
mflodmancc3d4422017-08-03 08:27:51 -07002636TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002638 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002639 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002640
sprangc5d62e22017-04-02 23:53:04 -07002641 const int kFrameWidth = 1280;
2642 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002643
Åsa Persson8c1bf952018-09-13 10:42:19 +02002644 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002645
kthelgason5e13d412016-12-01 03:59:51 -08002646 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002647 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002648 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002649 frame_timestamp += kFrameIntervalMs;
2650
perkj803d97f2016-11-01 11:45:46 -07002651 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002652 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002653 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002654 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002655 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002656 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002657
asapersson0944a802017-04-07 00:57:58 -07002658 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002659 // wanted resolution.
2660 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2661 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2662 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002663 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002664
2665 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002666 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002667 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002669 // Give the encoder queue time to process the change in degradation preference
2670 // by waiting for an encoded frame.
2671 new_video_source.IncomingCapturedFrame(
2672 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2673 sink_.WaitForEncodedFrame(frame_timestamp);
2674 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002675 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002676 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002677
sprangc5d62e22017-04-02 23:53:04 -07002678 // Force an input frame rate to be available, or the adaptation call won't
2679 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002680 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002681 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002682 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002683 stats_proxy_->SetMockStats(stats);
2684
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002686 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002687 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002688 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002689 frame_timestamp += kFrameIntervalMs;
2690
2691 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002692 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002693 EXPECT_EQ(std::numeric_limits<int>::max(),
2694 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002695 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002696
asapersson02465b82017-04-10 01:12:52 -07002697 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002698 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2699 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002700 // Give the encoder queue time to process the change in degradation preference
2701 // by waiting for an encoded frame.
2702 new_video_source.IncomingCapturedFrame(
2703 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2704 sink_.WaitForEncodedFrame(frame_timestamp);
2705 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002706 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002707
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002709 new_video_source.IncomingCapturedFrame(
2710 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002711 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002712 frame_timestamp += kFrameIntervalMs;
2713
2714 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002715 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002716
2717 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002718 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002719 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002720 // Give the encoder queue time to process the change in degradation preference
2721 // by waiting for an encoded frame.
2722 new_video_source.IncomingCapturedFrame(
2723 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2724 sink_.WaitForEncodedFrame(frame_timestamp);
2725 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002726 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2727 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002728 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002729 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002730
2731 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002732 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002733 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002734 // Give the encoder queue time to process the change in degradation preference
2735 // by waiting for an encoded frame.
2736 new_video_source.IncomingCapturedFrame(
2737 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2738 sink_.WaitForEncodedFrame(frame_timestamp);
2739 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002740 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2741 EXPECT_EQ(std::numeric_limits<int>::max(),
2742 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002743 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002744
mflodmancc3d4422017-08-03 08:27:51 -07002745 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002746}
2747
mflodmancc3d4422017-08-03 08:27:51 -07002748TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002750 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002751
asaperssonfab67072017-04-04 05:51:49 -07002752 const int kWidth = 1280;
2753 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002754 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002755 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002756 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2757 EXPECT_FALSE(stats.bw_limited_resolution);
2758 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2759
2760 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002761 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002762 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002763 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002764
2765 stats = stats_proxy_->GetStats();
2766 EXPECT_TRUE(stats.bw_limited_resolution);
2767 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2768
2769 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002770 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002771 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002772 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002773
2774 stats = stats_proxy_->GetStats();
2775 EXPECT_FALSE(stats.bw_limited_resolution);
2776 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2777 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2778
mflodmancc3d4422017-08-03 08:27:51 -07002779 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002780}
2781
mflodmancc3d4422017-08-03 08:27:51 -07002782TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002783 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002784 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002785
2786 const int kWidth = 1280;
2787 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002788 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002789 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002790 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2791 EXPECT_FALSE(stats.cpu_limited_resolution);
2792 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2793
2794 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002796 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002797 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002798
2799 stats = stats_proxy_->GetStats();
2800 EXPECT_TRUE(stats.cpu_limited_resolution);
2801 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2802
2803 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002804 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002805 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002806 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002807
2808 stats = stats_proxy_->GetStats();
2809 EXPECT_FALSE(stats.cpu_limited_resolution);
2810 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002811 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002812
mflodmancc3d4422017-08-03 08:27:51 -07002813 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002814}
2815
mflodmancc3d4422017-08-03 08:27:51 -07002816TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002819
asaperssonfab67072017-04-04 05:51:49 -07002820 const int kWidth = 1280;
2821 const int kHeight = 720;
2822 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002823 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002824 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002825 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002826 EXPECT_FALSE(stats.cpu_limited_resolution);
2827 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2828
asaperssonfab67072017-04-04 05:51:49 -07002829 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002830 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002831 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002832 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002833 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002834 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002835 EXPECT_TRUE(stats.cpu_limited_resolution);
2836 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2837
2838 // Set new source with adaptation still enabled.
2839 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002840 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002841 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002842
asaperssonfab67072017-04-04 05:51:49 -07002843 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002844 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002845 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002846 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002847 EXPECT_TRUE(stats.cpu_limited_resolution);
2848 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2849
2850 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002851 video_stream_encoder_->SetSource(&new_video_source,
2852 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002853
asaperssonfab67072017-04-04 05:51:49 -07002854 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002855 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002856 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002857 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002858 EXPECT_FALSE(stats.cpu_limited_resolution);
2859 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2860
2861 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002862 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002863 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002864
asaperssonfab67072017-04-04 05:51:49 -07002865 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002866 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002867 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002868 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002869 EXPECT_TRUE(stats.cpu_limited_resolution);
2870 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2871
asaperssonfab67072017-04-04 05:51:49 -07002872 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002873 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002874 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002875 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002876 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002877 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002878 EXPECT_FALSE(stats.cpu_limited_resolution);
2879 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002880 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002881
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002883}
2884
mflodmancc3d4422017-08-03 08:27:51 -07002885TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002886 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002887 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002888
asaperssonfab67072017-04-04 05:51:49 -07002889 const int kWidth = 1280;
2890 const int kHeight = 720;
2891 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002892 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002893 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002894 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002895 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002896 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002897
2898 // Set new source with adaptation still enabled.
2899 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002900 video_stream_encoder_->SetSource(&new_video_source,
2901 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002902
asaperssonfab67072017-04-04 05:51:49 -07002903 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002904 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002905 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002906 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002907 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002908 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002909
asaperssonfab67072017-04-04 05:51:49 -07002910 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002911 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002912 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002914 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002915 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002916 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002917 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002918
asaperssonfab67072017-04-04 05:51:49 -07002919 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002920 video_stream_encoder_->SetSource(&new_video_source,
2921 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002922
asaperssonfab67072017-04-04 05:51:49 -07002923 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002924 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002925 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002926 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002927 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002928 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002929
asapersson02465b82017-04-10 01:12:52 -07002930 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002931 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002932 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002933
asaperssonfab67072017-04-04 05:51:49 -07002934 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002935 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002936 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002937 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002938 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002939 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2940 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002941
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002943}
2944
mflodmancc3d4422017-08-03 08:27:51 -07002945TEST_F(VideoStreamEncoderTest,
2946 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002948 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002949
2950 const int kWidth = 1280;
2951 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002952 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002953 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002954 video_source_.IncomingCapturedFrame(
2955 CreateFrame(timestamp_ms, kWidth, kHeight));
2956 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002957 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2958 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2959 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2960
2961 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002962 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002963 timestamp_ms += kFrameIntervalMs;
2964 video_source_.IncomingCapturedFrame(
2965 CreateFrame(timestamp_ms, kWidth, kHeight));
2966 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002967 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2969 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2970
2971 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002973 timestamp_ms += kFrameIntervalMs;
2974 video_source_.IncomingCapturedFrame(
2975 CreateFrame(timestamp_ms, kWidth, kHeight));
2976 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002977 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2978 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2979 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2980
Niels Möller4db138e2018-04-19 09:04:13 +02002981 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002982 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002983
2984 VideoEncoderConfig video_encoder_config;
2985 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2986 // Make format different, to force recreation of encoder.
2987 video_encoder_config.video_format.parameters["foo"] = "foo";
2988 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002989 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002990 timestamp_ms += kFrameIntervalMs;
2991 video_source_.IncomingCapturedFrame(
2992 CreateFrame(timestamp_ms, kWidth, kHeight));
2993 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002994 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2995 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2996 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2997
mflodmancc3d4422017-08-03 08:27:51 -07002998 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002999}
3000
mflodmancc3d4422017-08-03 08:27:51 -07003001TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003002 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003004 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003005
3006 const int kWidth = 1280;
3007 const int kHeight = 720;
3008 int sequence = 1;
3009
3010 // Enable BALANCED preference, no initial limitation.
3011 test::FrameForwarder source;
3012 video_stream_encoder_->SetSource(&source,
3013 webrtc::DegradationPreference::BALANCED);
3014 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3015 WaitForEncodedFrame(sequence++);
3016 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3017 EXPECT_FALSE(stats.cpu_limited_resolution);
3018 EXPECT_FALSE(stats.cpu_limited_framerate);
3019 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3020
3021 // Trigger CPU overuse, should now adapt down.
3022 video_stream_encoder_->TriggerCpuOveruse();
3023 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3024 WaitForEncodedFrame(sequence++);
3025 stats = stats_proxy_->GetStats();
3026 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3027
3028 // Set new degradation preference should clear restrictions since we changed
3029 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003030 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003031 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3032 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3033 WaitForEncodedFrame(sequence++);
3034 stats = stats_proxy_->GetStats();
3035 EXPECT_FALSE(stats.cpu_limited_resolution);
3036 EXPECT_FALSE(stats.cpu_limited_framerate);
3037 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3038
3039 // Force an input frame rate to be available, or the adaptation call won't
3040 // know what framerate to adapt from.
3041 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3042 mock_stats.input_frame_rate = 30;
3043 stats_proxy_->SetMockStats(mock_stats);
3044 video_stream_encoder_->TriggerCpuOveruse();
3045 stats_proxy_->ResetMockStats();
3046 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3047 WaitForEncodedFrame(sequence++);
3048
3049 // We have now adapted once.
3050 stats = stats_proxy_->GetStats();
3051 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3052
3053 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003054 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3055 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003056 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3057 WaitForEncodedFrame(sequence++);
3058 stats = stats_proxy_->GetStats();
3059 EXPECT_FALSE(stats.cpu_limited_resolution);
3060 EXPECT_FALSE(stats.cpu_limited_framerate);
3061 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3062
3063 video_stream_encoder_->Stop();
3064}
3065
3066TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003067 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003069 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003070
asapersson0944a802017-04-07 00:57:58 -07003071 const int kWidth = 1280;
3072 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003073 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003074
asaperssonfab67072017-04-04 05:51:49 -07003075 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003076 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003077 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003078 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003079 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003080 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3081
asapersson02465b82017-04-10 01:12:52 -07003082 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003083 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003084 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003085 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003086 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003087 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003088 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003089 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3090
3091 // Set new source with adaptation still enabled.
3092 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003093 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003094 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003095
3096 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003097 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003098 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003099 stats = stats_proxy_->GetStats();
3100 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003101 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003102 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3103
sprangc5d62e22017-04-02 23:53:04 -07003104 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003105 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003106 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003107 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003108 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003109 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003110 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003111 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003112 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003113 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003114 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3115
sprangc5d62e22017-04-02 23:53:04 -07003116 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003117 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003118 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3119 mock_stats.input_frame_rate = 30;
3120 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003121 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003122 stats_proxy_->ResetMockStats();
3123
3124 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003125 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003126 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003127
3128 // Framerate now adapted.
3129 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003130 EXPECT_FALSE(stats.cpu_limited_resolution);
3131 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003132 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3133
3134 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003135 video_stream_encoder_->SetSource(&new_video_source,
3136 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003137 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003138 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003139 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003140
3141 stats = stats_proxy_->GetStats();
3142 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003143 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003144 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3145
3146 // Try to trigger overuse. Should not succeed.
3147 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003148 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003149 stats_proxy_->ResetMockStats();
3150
3151 stats = stats_proxy_->GetStats();
3152 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003154 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3155
3156 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003157 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003158 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003159 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003160 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003161 stats = stats_proxy_->GetStats();
3162 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003163 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003164 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003165
3166 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003167 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003168 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003169 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003170 stats = stats_proxy_->GetStats();
3171 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003172 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003173 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3174
3175 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003176 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003177 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003178 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003179 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003180 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003181 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003182 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003183 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003184 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003185 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3186
3187 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003188 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003189 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003190 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003191 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003192 stats = stats_proxy_->GetStats();
3193 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003194 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003195 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003196 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003197
mflodmancc3d4422017-08-03 08:27:51 -07003198 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003199}
3200
mflodmancc3d4422017-08-03 08:27:51 -07003201TEST_F(VideoStreamEncoderTest,
3202 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003203 const int kWidth = 1280;
3204 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003206 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003207
asaperssonfab67072017-04-04 05:51:49 -07003208 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003209 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003210
asaperssonfab67072017-04-04 05:51:49 -07003211 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003212 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003213
asaperssonfab67072017-04-04 05:51:49 -07003214 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003216
asaperssonfab67072017-04-04 05:51:49 -07003217 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003218 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003219
kthelgason876222f2016-11-29 01:44:11 -08003220 // Expect a scale down.
3221 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003222 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003223
asapersson02465b82017-04-10 01:12:52 -07003224 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003225 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003226 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003227 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003228
asaperssonfab67072017-04-04 05:51:49 -07003229 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003230 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003231 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003232 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003233
asaperssonfab67072017-04-04 05:51:49 -07003234 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003235 EXPECT_EQ(std::numeric_limits<int>::max(),
3236 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003237
asaperssonfab67072017-04-04 05:51:49 -07003238 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003239 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003240 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003241 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003242
asapersson02465b82017-04-10 01:12:52 -07003243 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003244 EXPECT_EQ(std::numeric_limits<int>::max(),
3245 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003246
mflodmancc3d4422017-08-03 08:27:51 -07003247 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003248}
3249
mflodmancc3d4422017-08-03 08:27:51 -07003250TEST_F(VideoStreamEncoderTest,
3251 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003252 const int kWidth = 1280;
3253 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003255 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003256
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003257 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003258 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003260 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003261
3262 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003263 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003264 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003265 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3266 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3267
3268 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003269 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003270 EXPECT_THAT(source.sink_wants(),
3271 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003272 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3273 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3274 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3275
3276 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003277 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003278 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3279 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3280 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3281
mflodmancc3d4422017-08-03 08:27:51 -07003282 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003283}
3284
mflodmancc3d4422017-08-03 08:27:51 -07003285TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003286 const int kWidth = 1280;
3287 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003289 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003290
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003291 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003292 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003293 video_stream_encoder_->SetSource(&source,
3294 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003295 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3296 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003297 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003298
3299 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003301 EXPECT_THAT(source.sink_wants(),
3302 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003303 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3304 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3305 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3306
3307 // Trigger adapt down for same input resolution, expect no change.
3308 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3309 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003310 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003311 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3312 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3313 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3314
3315 // Trigger adapt down for larger input resolution, expect no change.
3316 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3317 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003319 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3320 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3321 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3322
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003324}
3325
mflodmancc3d4422017-08-03 08:27:51 -07003326TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003327 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3328 const int kWidth = 640;
3329 const int kHeight = 360;
3330 const int64_t kFrameIntervalMs = 150;
3331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003332 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003333
3334 // Enable BALANCED preference, no initial limitation.
3335 AdaptingFrameForwarder source(&time_controller_);
3336 source.set_adaptation_enabled(true);
3337 video_stream_encoder_->SetSource(&source,
3338 webrtc::DegradationPreference::BALANCED);
3339
3340 int64_t timestamp_ms = kFrameIntervalMs;
3341 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3342 sink_.WaitForEncodedFrame(kWidth, kHeight);
3343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3346 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3347
3348 // Trigger adapt down, expect reduced fps (640x360@15fps).
3349 video_stream_encoder_->TriggerQualityLow();
3350 timestamp_ms += kFrameIntervalMs;
3351 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3352 sink_.WaitForEncodedFrame(timestamp_ms);
3353 EXPECT_THAT(source.sink_wants(),
3354 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3356 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3357 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3358
3359 // Source requests 270p, expect reduced resolution (480x270@15fps).
3360 source.OnOutputFormatRequest(480, 270);
3361 timestamp_ms += kFrameIntervalMs;
3362 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3363 WaitForEncodedFrame(480, 270);
3364 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3365
3366 // Trigger adapt down, expect reduced fps (480x270@10fps).
3367 video_stream_encoder_->TriggerQualityLow();
3368 timestamp_ms += kFrameIntervalMs;
3369 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3370 sink_.WaitForEncodedFrame(timestamp_ms);
3371 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3372 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3373 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3374 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3375
3376 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3377 source.OnOutputFormatRequest(320, 180);
3378 timestamp_ms += kFrameIntervalMs;
3379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3380 WaitForEncodedFrame(320, 180);
3381 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt down, expect reduced fps (320x180@7fps).
3384 video_stream_encoder_->TriggerQualityLow();
3385 timestamp_ms += kFrameIntervalMs;
3386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3387 sink_.WaitForEncodedFrame(timestamp_ms);
3388 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3391 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3392
3393 // Source requests VGA, expect increased resolution (640x360@7fps).
3394 source.OnOutputFormatRequest(640, 360);
3395 timestamp_ms += kFrameIntervalMs;
3396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3397 WaitForEncodedFrame(timestamp_ms);
3398 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3401 video_stream_encoder_->TriggerQualityHigh();
3402 timestamp_ms += kFrameIntervalMs;
3403 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3404 WaitForEncodedFrame(timestamp_ms);
3405 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3407 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3408 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3409
3410 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3411 video_stream_encoder_->TriggerQualityHigh();
3412 timestamp_ms += kFrameIntervalMs;
3413 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3414 WaitForEncodedFrame(timestamp_ms);
3415 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3416 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3418 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419
3420 // Trigger adapt up, expect increased fps (640x360@maxfps).
3421 video_stream_encoder_->TriggerQualityHigh();
3422 timestamp_ms += kFrameIntervalMs;
3423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3424 WaitForEncodedFrame(timestamp_ms);
3425 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3426 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3428 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
3430 video_stream_encoder_->Stop();
3431}
3432
3433TEST_F(VideoStreamEncoderTest,
3434 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3435 const int kWidth = 1280;
3436 const int kHeight = 720;
3437 const int64_t kFrameIntervalMs = 150;
3438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003439 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003440
3441 // Enable BALANCED preference, no initial limitation.
3442 AdaptingFrameForwarder source(&time_controller_);
3443 source.set_adaptation_enabled(true);
3444 video_stream_encoder_->SetSource(&source,
3445 webrtc::DegradationPreference::BALANCED);
3446
3447 int64_t timestamp_ms = kFrameIntervalMs;
3448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3449 sink_.WaitForEncodedFrame(kWidth, kHeight);
3450 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3453 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3456 video_stream_encoder_->TriggerQualityLow();
3457 timestamp_ms += kFrameIntervalMs;
3458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3459 sink_.WaitForEncodedFrame(timestamp_ms);
3460 EXPECT_THAT(source.sink_wants(),
3461 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3462 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3464 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3465
3466 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3467 video_stream_encoder_->TriggerQualityLow();
3468 timestamp_ms += kFrameIntervalMs;
3469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3470 sink_.WaitForEncodedFrame(timestamp_ms);
3471 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3474 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475
3476 // Trigger adapt down, expect reduced fps (640x360@15fps).
3477 video_stream_encoder_->TriggerQualityLow();
3478 timestamp_ms += kFrameIntervalMs;
3479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3480 WaitForEncodedFrame(timestamp_ms);
3481 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3484 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3487 source.OnOutputFormatRequest(320, 180);
3488 timestamp_ms += kFrameIntervalMs;
3489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3490 WaitForEncodedFrame(320, 180);
3491 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3493
3494 // Trigger adapt down, expect reduced fps (320x180@7fps).
3495 video_stream_encoder_->TriggerCpuOveruse();
3496 timestamp_ms += kFrameIntervalMs;
3497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3498 WaitForEncodedFrame(timestamp_ms);
3499 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3501 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3502 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3503 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3504 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3505 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3506
3507 // Source requests HD, expect increased resolution (640x360@7fps).
3508 source.OnOutputFormatRequest(1280, 720);
3509 timestamp_ms += kFrameIntervalMs;
3510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3511 WaitForEncodedFrame(timestamp_ms);
3512 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3513 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3514
3515 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3516 video_stream_encoder_->TriggerCpuUnderuse();
3517 timestamp_ms += kFrameIntervalMs;
3518 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3519 WaitForEncodedFrame(timestamp_ms);
3520 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3523 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3525 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3526 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3527
3528 // Trigger adapt up, expect increased fps (640x360@maxfps).
3529 video_stream_encoder_->TriggerQualityHigh();
3530 video_stream_encoder_->TriggerCpuUnderuse();
3531 timestamp_ms += kFrameIntervalMs;
3532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3533 WaitForEncodedFrame(timestamp_ms);
3534 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3537 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3538 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3539 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3540 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3541
3542 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3543 video_stream_encoder_->TriggerQualityHigh();
3544 video_stream_encoder_->TriggerCpuUnderuse();
3545 timestamp_ms += kFrameIntervalMs;
3546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3547 WaitForEncodedFrame(timestamp_ms);
3548 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3551 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3553 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3554 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3555
3556 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3557 video_stream_encoder_->TriggerQualityHigh();
3558 video_stream_encoder_->TriggerCpuUnderuse();
3559 timestamp_ms += kFrameIntervalMs;
3560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3561 WaitForEncodedFrame(timestamp_ms);
3562 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3564 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3565 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3566 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3567 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3568 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3569
3570 video_stream_encoder_->Stop();
3571}
3572
3573TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003574 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003575 const int kWidth = 1280;
3576 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003578 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003579
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003580 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003581 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003582 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003583 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003584
3585 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003586 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003587 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003588 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3589 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3590
3591 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003592 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003593 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003594 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3595 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3596
mflodmancc3d4422017-08-03 08:27:51 -07003597 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003598}
3599
mflodmancc3d4422017-08-03 08:27:51 -07003600TEST_F(VideoStreamEncoderTest,
3601 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003602 const int kWidth = 1280;
3603 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003605 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003606
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003607 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003608 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003609 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003610 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003611
3612 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003613 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003614 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003616 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3617
3618 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003619 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003620 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003621 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003622 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3623
mflodmancc3d4422017-08-03 08:27:51 -07003624 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003625}
3626
mflodmancc3d4422017-08-03 08:27:51 -07003627TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003628 const int kWidth = 1280;
3629 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003631 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003632
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003633 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003634 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003635 video_stream_encoder_->SetSource(&source,
3636 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003637
3638 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3639 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003640 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003641 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3642 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3643 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3644
3645 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003646 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003647 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3650 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3651
mflodmancc3d4422017-08-03 08:27:51 -07003652 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003653}
3654
mflodmancc3d4422017-08-03 08:27:51 -07003655TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003656 const int kWidth = 1280;
3657 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003659 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003660
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003661 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003662 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003663 video_stream_encoder_->SetSource(&source,
3664 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003665
3666 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3667 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003668 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3670 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3671 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3672
3673 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003674 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3679
mflodmancc3d4422017-08-03 08:27:51 -07003680 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003681}
3682
mflodmancc3d4422017-08-03 08:27:51 -07003683TEST_F(VideoStreamEncoderTest,
3684 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003685 const int kWidth = 1280;
3686 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003688 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003689
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003690 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003691 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003692 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003693 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003694 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003695
3696 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003697 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003698 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003699 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3700 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3701
3702 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003704 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003705 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003706 EXPECT_THAT(source.sink_wants(),
3707 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003708 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3709 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3710
3711 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003712 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003713 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003714 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3715 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3716 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3717
mflodmancc3d4422017-08-03 08:27:51 -07003718 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003719}
3720
mflodmancc3d4422017-08-03 08:27:51 -07003721TEST_F(VideoStreamEncoderTest,
3722 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003723 const int kWidth = 1280;
3724 const int kHeight = 720;
3725 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003727 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003728
3729 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3730 stats.input_frame_rate = kInputFps;
3731 stats_proxy_->SetMockStats(stats);
3732
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003733 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003734 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3735 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003736 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003737
3738 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003739 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003740 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3741 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003742 EXPECT_THAT(video_source_.sink_wants(),
3743 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003744
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003745 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003746 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003747 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003748 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003749 // Give the encoder queue time to process the change in degradation preference
3750 // by waiting for an encoded frame.
3751 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3752 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003753 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003754
3755 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003756 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003757 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3758 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(new_video_source.sink_wants(),
3760 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003761
3762 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003763 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003764 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003765
mflodmancc3d4422017-08-03 08:27:51 -07003766 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003767}
3768
mflodmancc3d4422017-08-03 08:27:51 -07003769TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003770 const int kWidth = 1280;
3771 const int kHeight = 720;
3772 const size_t kNumFrames = 10;
3773
Henrik Boström381d1092020-05-12 18:49:07 +02003774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003776
asaperssond0de2952017-04-21 01:47:31 -07003777 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003778 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003779 video_source_.set_adaptation_enabled(true);
3780
3781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3782 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3783
3784 int downscales = 0;
3785 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003786 video_source_.IncomingCapturedFrame(
3787 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3788 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003789
asaperssonfab67072017-04-04 05:51:49 -07003790 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003791 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003792 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003793 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003794
3795 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3796 ++downscales;
3797
3798 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3799 EXPECT_EQ(downscales,
3800 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3801 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003802 }
mflodmancc3d4422017-08-03 08:27:51 -07003803 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003804}
3805
mflodmancc3d4422017-08-03 08:27:51 -07003806TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003807 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3808 const int kWidth = 1280;
3809 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003811 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003812
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003813 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003814 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003815 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003816 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003817 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003818
Åsa Persson8c1bf952018-09-13 10:42:19 +02003819 int64_t timestamp_ms = kFrameIntervalMs;
3820 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003821 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003822 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003823 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3824 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3825
3826 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003827 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003828 timestamp_ms += kFrameIntervalMs;
3829 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3830 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003831 EXPECT_THAT(source.sink_wants(),
3832 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003833 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3834 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3835
3836 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003837 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003838 timestamp_ms += kFrameIntervalMs;
3839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003840 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3843 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3844
3845 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003846 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003847 timestamp_ms += kFrameIntervalMs;
3848 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3849 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003850 EXPECT_THAT(source.sink_wants(),
3851 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003852 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3853 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3854
3855 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003856 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003857 timestamp_ms += kFrameIntervalMs;
3858 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003859 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003860 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003861 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3862 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3863
mflodmancc3d4422017-08-03 08:27:51 -07003864 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003865}
3866
mflodmancc3d4422017-08-03 08:27:51 -07003867TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003868 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3869 const int kWidth = 1280;
3870 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003873
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003874 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003875 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003876 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003877 video_stream_encoder_->SetSource(&source,
3878 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003879
Åsa Persson8c1bf952018-09-13 10:42:19 +02003880 int64_t timestamp_ms = kFrameIntervalMs;
3881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003882 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003883 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003884 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3885 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3886
3887 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003888 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003889 timestamp_ms += kFrameIntervalMs;
3890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3891 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003892 EXPECT_THAT(source.sink_wants(),
3893 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003894 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3896
3897 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003898 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003899 timestamp_ms += kFrameIntervalMs;
3900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003901 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003902 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3904 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3905
3906 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003907 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003908 timestamp_ms += kFrameIntervalMs;
3909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3910 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003911 EXPECT_THAT(source.sink_wants(),
3912 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003913 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3914 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3915
3916 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003917 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003918 timestamp_ms += kFrameIntervalMs;
3919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003920 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003921 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003922 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3923 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3924
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003926}
3927
Sergey Silkin41c650b2019-10-14 13:12:19 +02003928TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3929 fake_encoder_.SetResolutionBitrateLimits(
3930 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3931
Henrik Boström381d1092020-05-12 18:49:07 +02003932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003933 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3934 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3935 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3936 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003937
3938 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003939 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003940 source.set_adaptation_enabled(true);
3941 video_stream_encoder_->SetSource(
3942 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3943
3944 // Insert 720p frame.
3945 int64_t timestamp_ms = kFrameIntervalMs;
3946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3947 WaitForEncodedFrame(1280, 720);
3948
3949 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003951 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3952 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3953 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3954 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003955 video_stream_encoder_->TriggerQualityLow();
3956
3957 // Insert 720p frame. It should be downscaled and encoded.
3958 timestamp_ms += kFrameIntervalMs;
3959 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3960 WaitForEncodedFrame(960, 540);
3961
3962 // Trigger adapt up. Higher resolution should not be requested duo to lack
3963 // of bitrate.
3964 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003965 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003966
3967 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003969 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3970 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3971 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3972 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003973
3974 // Trigger adapt up. Higher resolution should be requested.
3975 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003976 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003977
3978 video_stream_encoder_->Stop();
3979}
3980
3981TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3982 fake_encoder_.SetResolutionBitrateLimits(
3983 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3984
3985 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003987 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3988 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3989 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3990 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003991
3992 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003993 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003994 source.set_adaptation_enabled(true);
3995 video_stream_encoder_->SetSource(
3996 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3997
3998 // Insert 720p frame. It should be dropped and lower resolution should be
3999 // requested.
4000 int64_t timestamp_ms = kFrameIntervalMs;
4001 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4002 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004003 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004004
4005 // Insert 720p frame. It should be downscaled and encoded.
4006 timestamp_ms += kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4008 WaitForEncodedFrame(960, 540);
4009
4010 video_stream_encoder_->Stop();
4011}
4012
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004013class BalancedDegradationTest : public VideoStreamEncoderTest {
4014 protected:
4015 void SetupTest() {
4016 // Reset encoder for field trials to take effect.
4017 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004018 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004019
4020 // Enable BALANCED preference.
4021 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004022 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4023 }
4024
Asa Persson606d3cb2021-10-04 10:07:11 +02004025 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004027 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004028 }
4029
Åsa Persson45b176f2019-09-30 11:19:05 +02004030 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004031 timestamp_ms_ += kFrameIntervalMs;
4032 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004033 }
4034
4035 void InsertFrameAndWaitForEncoded() {
4036 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004037 sink_.WaitForEncodedFrame(timestamp_ms_);
4038 }
4039
4040 const int kWidth = 640; // pixels:640x360=230400
4041 const int kHeight = 360;
4042 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4043 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004044 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004045};
4046
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004047TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004048 test::ScopedFieldTrials field_trials(
4049 "WebRTC-Video-BalancedDegradationSettings/"
4050 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4051 SetupTest();
4052
4053 // Force input frame rate.
4054 const int kInputFps = 24;
4055 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4056 stats.input_frame_rate = kInputFps;
4057 stats_proxy_->SetMockStats(stats);
4058
Åsa Persson45b176f2019-09-30 11:19:05 +02004059 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004060 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004061
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004062 // Trigger adapt down, expect scaled down framerate and resolution,
4063 // since Fps diff (input-requested:0) < threshold.
4064 video_stream_encoder_->TriggerQualityLow();
4065 EXPECT_THAT(source_.sink_wants(),
4066 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004067
4068 video_stream_encoder_->Stop();
4069}
4070
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004071TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004072 test::ScopedFieldTrials field_trials(
4073 "WebRTC-Video-BalancedDegradationSettings/"
4074 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4075 SetupTest();
4076
4077 // Force input frame rate.
4078 const int kInputFps = 25;
4079 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4080 stats.input_frame_rate = kInputFps;
4081 stats_proxy_->SetMockStats(stats);
4082
Åsa Persson45b176f2019-09-30 11:19:05 +02004083 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004084 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004085
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004086 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4087 // Fps diff (input-requested:1) == threshold.
4088 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004089 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004090
4091 video_stream_encoder_->Stop();
4092}
4093
4094TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4095 test::ScopedFieldTrials field_trials(
4096 "WebRTC-Video-BalancedDegradationSettings/"
4097 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4098 SetupTest();
4099
4100 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4101
Åsa Persson45b176f2019-09-30 11:19:05 +02004102 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004103 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004104
4105 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4106 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004107 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004108
4109 video_stream_encoder_->Stop();
4110}
4111
Åsa Perssonccfb3402019-09-25 15:13:04 +02004112TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004113 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004114 "WebRTC-Video-BalancedDegradationSettings/"
4115 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004116 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004117
Asa Persson606d3cb2021-10-04 10:07:11 +02004118 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4119 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4120 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004121
Åsa Persson45b176f2019-09-30 11:19:05 +02004122 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004123 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004124 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4125
4126 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4127 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004128 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004129 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004130 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4131
4132 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4133 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004134 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004135 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004136 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4137
Åsa Persson30ab0152019-08-27 12:22:33 +02004138 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4139 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004140 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004141 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004142 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004143 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4144
4145 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004146 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004147 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004148 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004149
Åsa Persson30ab0152019-08-27 12:22:33 +02004150 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004151 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004152 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004153 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004154 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004155 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4156
4157 video_stream_encoder_->Stop();
4158}
4159
Åsa Perssonccfb3402019-09-25 15:13:04 +02004160TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004161 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4162 test::ScopedFieldTrials field_trials(
4163 "WebRTC-Video-BalancedDegradationSettings/"
4164 "pixels:57600|129600|230400,fps:7|24|24/");
4165 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004166 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004167
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004168 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004169
4170 // Insert frame, expect scaled down:
4171 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4172 InsertFrame();
4173 EXPECT_FALSE(WaitForFrame(1000));
4174 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4175 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4176
4177 // Insert frame, expect scaled down:
4178 // resolution (320x180@24fps).
4179 InsertFrame();
4180 EXPECT_FALSE(WaitForFrame(1000));
4181 EXPECT_LT(source_.sink_wants().max_pixel_count,
4182 source_.last_wants().max_pixel_count);
4183 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4184
4185 // Frame should not be dropped (min pixels per frame reached).
4186 InsertFrameAndWaitForEncoded();
4187
4188 video_stream_encoder_->Stop();
4189}
4190
4191TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004192 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004193 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004194 "WebRTC-Video-BalancedDegradationSettings/"
4195 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004196 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004197
Asa Persson606d3cb2021-10-04 10:07:11 +02004198 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4199 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4200 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004201
Åsa Persson45b176f2019-09-30 11:19:05 +02004202 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004203 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004204 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4207 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004209 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004210 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4211
4212 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004215 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004216 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4217
4218 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4219 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004220 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004221 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004222 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4223
Åsa Persson30ab0152019-08-27 12:22:33 +02004224 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4225 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004226 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004227 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004228 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4229
4230 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4231 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004232 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004233 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4234
4235 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004236 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004237 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004238 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004239 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4241
4242 video_stream_encoder_->Stop();
4243}
4244
Åsa Perssonccfb3402019-09-25 15:13:04 +02004245TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004246 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004247 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004248 "WebRTC-Video-BalancedDegradationSettings/"
4249 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004250 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004251
Asa Persson606d3cb2021-10-04 10:07:11 +02004252 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4253 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4254 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4255 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4256 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004257
Åsa Persson45b176f2019-09-30 11:19:05 +02004258 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004259 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004260 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4261
4262 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4263 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004264 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004265 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004266 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4267
4268 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4269 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004270 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004271 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004272 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4273
4274 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4275 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004276 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004277 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004278 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4279
4280 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4281 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004282 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004283 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4284
4285 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004286 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004287 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004288 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004289 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004290 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4291
4292 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004293 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004295 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004296 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4297
4298 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004299 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004300 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004301 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004302 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004303 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4304
Åsa Persson1b247f12019-08-14 17:26:39 +02004305 video_stream_encoder_->Stop();
4306}
4307
mflodmancc3d4422017-08-03 08:27:51 -07004308TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004309 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4310 const int kWidth = 1280;
4311 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004313 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004314
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004315 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004316 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004317 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004318 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004319 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004320
Åsa Persson8c1bf952018-09-13 10:42:19 +02004321 int64_t timestamp_ms = kFrameIntervalMs;
4322 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004323 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004324 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004325 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4326 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4327 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4328 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4329
4330 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004331 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004332 timestamp_ms += kFrameIntervalMs;
4333 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4334 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004335 EXPECT_THAT(source.sink_wants(),
4336 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004337 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4338 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4339 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4341
4342 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004343 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004344 timestamp_ms += kFrameIntervalMs;
4345 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4346 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004347 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004348 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4349 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4350 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4351 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4352
Jonathan Yubc771b72017-12-08 17:04:29 -08004353 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004354 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004355 timestamp_ms += kFrameIntervalMs;
4356 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4357 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004358 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004359 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4360 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004361 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004362 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4363
Jonathan Yubc771b72017-12-08 17:04:29 -08004364 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004365 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004366 timestamp_ms += kFrameIntervalMs;
4367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4368 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004369 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004370 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004371 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4373 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4374 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4375
Jonathan Yubc771b72017-12-08 17:04:29 -08004376 // Trigger quality adapt down, expect no change (min resolution reached).
4377 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004378 timestamp_ms += kFrameIntervalMs;
4379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4380 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004381 EXPECT_THAT(source.sink_wants(), FpsMax());
4382 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004383 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4384 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4385 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4386 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4387
Evan Shrubsole64469032020-06-11 10:45:29 +02004388 // Trigger quality adapt up, expect upscaled resolution (480x270).
4389 video_stream_encoder_->TriggerQualityHigh();
4390 timestamp_ms += kFrameIntervalMs;
4391 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4392 WaitForEncodedFrame(timestamp_ms);
4393 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4394 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4395 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4396 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4397 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4398
4399 // Trigger quality and cpu adapt up since both are most limited, expect
4400 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004401 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004402 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004403 timestamp_ms += kFrameIntervalMs;
4404 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4405 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004406 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004407 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4408 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4409 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004410 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004411
Evan Shrubsole64469032020-06-11 10:45:29 +02004412 // Trigger quality and cpu adapt up since both are most limited, expect
4413 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004414 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004415 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004416 timestamp_ms += kFrameIntervalMs;
4417 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4418 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004420 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004421 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004423 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4424 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004425
Evan Shrubsole64469032020-06-11 10:45:29 +02004426 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4427 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004428 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004429 timestamp_ms += kFrameIntervalMs;
4430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4431 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004432 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004433 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004435 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004436 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004437
4438 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004439 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004440 timestamp_ms += kFrameIntervalMs;
4441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004442 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004443 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004444 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004445 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004447 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004448 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004449
mflodmancc3d4422017-08-03 08:27:51 -07004450 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004451}
4452
mflodmancc3d4422017-08-03 08:27:51 -07004453TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004454 const int kWidth = 640;
4455 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004456
Henrik Boström381d1092020-05-12 18:49:07 +02004457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004458 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004459
perkj803d97f2016-11-01 11:45:46 -07004460 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004461 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004462 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004463 }
4464
mflodmancc3d4422017-08-03 08:27:51 -07004465 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004466 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004467 video_source_.IncomingCapturedFrame(CreateFrame(
4468 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004469 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004470 }
4471
mflodmancc3d4422017-08-03 08:27:51 -07004472 video_stream_encoder_->Stop();
4473 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004474 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004475
Ying Wangef3998f2019-12-09 13:06:53 +01004476 EXPECT_METRIC_EQ(
4477 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4478 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004479 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4480}
4481
mflodmancc3d4422017-08-03 08:27:51 -07004482TEST_F(VideoStreamEncoderTest,
4483 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004485 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004486 const int kWidth = 640;
4487 const int kHeight = 360;
4488
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004489 video_stream_encoder_->SetSource(&video_source_,
4490 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004491
4492 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4493 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004494 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004495 }
4496
mflodmancc3d4422017-08-03 08:27:51 -07004497 video_stream_encoder_->Stop();
4498 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004499 stats_proxy_.reset();
4500
4501 EXPECT_EQ(0,
4502 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4503}
4504
Per Kjellanderdcef6412020-10-07 15:09:05 +02004505TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4506 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004507 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004508 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004509
4510 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004511 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004512 SimulcastRateAllocator(fake_encoder_.config())
4513 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004514 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004515
Henrik Boström381d1092020-05-12 18:49:07 +02004516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004517 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004518
sprang57c2fff2017-01-16 06:24:02 -08004519 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004520 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4521 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004522 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4523 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4524
Erik Språngd7329ca2019-02-21 21:19:53 +01004525 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004526 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004527 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004528
Per Kjellanderdcef6412020-10-07 15:09:05 +02004529 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004530 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004531 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4532 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004533 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004534 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004535
Per Kjellanderdcef6412020-10-07 15:09:05 +02004536 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004537 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004538 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004539 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004540 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4541 WaitForEncodedFrame(CurrentTimeMs());
4542 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004543 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004544 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004545
mflodmancc3d4422017-08-03 08:27:51 -07004546 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004547}
4548
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004549TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004550 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004551 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004552 kVideoLayersAllocation);
4553
4554 const int kDefaultFps = 30;
4555
4556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004557 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004558
4559 video_source_.IncomingCapturedFrame(
4560 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4561 WaitForEncodedFrame(CurrentTimeMs());
4562 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4563 VideoLayersAllocation last_layer_allocation =
4564 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004565 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004566 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4567
4568 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004569 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004570 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004571 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004572 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4573
Erik Språng9d69cbe2020-10-22 17:44:42 +02004574 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004575 int number_of_layers_allocation = 1;
4576 const int64_t start_time_ms = CurrentTimeMs();
4577 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4578 video_source_.IncomingCapturedFrame(
4579 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4580 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004581 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4582 number_of_layers_allocation = sink_.number_of_layers_allocations();
4583 VideoLayersAllocation new_allocation =
4584 sink_.GetLastVideoLayersAllocation();
4585 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4586 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4587 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4588 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4589 .target_bitrate_per_temporal_layer,
4590 last_layer_allocation.active_spatial_layers[0]
4591 .target_bitrate_per_temporal_layer);
4592 last_layer_allocation = new_allocation;
4593 }
4594 }
4595 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4596 video_stream_encoder_->Stop();
4597}
4598
4599TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004600 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004601 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4602 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4603 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004604 VideoEncoderConfig video_encoder_config;
4605 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4606 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004607 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004608 video_encoder_config.content_type =
4609 VideoEncoderConfig::ContentType::kRealtimeVideo;
4610 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004611 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004612 VideoEncoder::GetDefaultVp8Settings());
4613 for (auto& layer : video_encoder_config.simulcast_layers) {
4614 layer.num_temporal_layers = 2;
4615 }
4616 // Simulcast layers are used for enabling/disabling streams.
4617 video_encoder_config.simulcast_layers[0].active = true;
4618 video_encoder_config.simulcast_layers[1].active = false;
4619 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004620 ConfigureEncoder(std::move(video_encoder_config),
4621 VideoStreamEncoder::BitrateAllocationCallbackType::
4622 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004623
4624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004625 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004626
4627 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4628 WaitForEncodedFrame(CurrentTimeMs());
4629 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4630 VideoLayersAllocation last_layer_allocation =
4631 sink_.GetLastVideoLayersAllocation();
4632
4633 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4634 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4635 .target_bitrate_per_temporal_layer,
4636 SizeIs(2));
4637 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4638 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4639 video_stream_encoder_->Stop();
4640}
4641
4642TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004643 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004644 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4645 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4646 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004647 VideoEncoderConfig video_encoder_config;
4648 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4649 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004650 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004651 video_encoder_config.content_type =
4652 VideoEncoderConfig::ContentType::kRealtimeVideo;
4653 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004654 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004655 VideoEncoder::GetDefaultVp8Settings());
4656 for (auto& layer : video_encoder_config.simulcast_layers) {
4657 layer.num_temporal_layers = 2;
4658 }
4659 // Simulcast layers are used for enabling/disabling streams.
4660 video_encoder_config.simulcast_layers[0].active = true;
4661 video_encoder_config.simulcast_layers[1].active = false;
4662 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004663 ConfigureEncoder(std::move(video_encoder_config),
4664 VideoStreamEncoder::BitrateAllocationCallbackType::
4665 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666
4667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004668 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004669
4670 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4671 WaitForEncodedFrame(CurrentTimeMs());
4672 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4673 VideoLayersAllocation last_layer_allocation =
4674 sink_.GetLastVideoLayersAllocation();
4675
4676 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4677 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4678 .target_bitrate_per_temporal_layer,
4679 SizeIs(2));
4680 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4681
4682 video_stream_encoder_->Stop();
4683}
4684
4685TEST_F(VideoStreamEncoderTest,
4686 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4687 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4688 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004689 VideoEncoderConfig video_encoder_config;
4690 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4691 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004692 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 video_encoder_config.content_type =
4694 VideoEncoderConfig::ContentType::kRealtimeVideo;
4695 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4696 vp9_settings.numberOfSpatialLayers = 2;
4697 vp9_settings.numberOfTemporalLayers = 2;
4698 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4699 vp9_settings.automaticResizeOn = false;
4700 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004701 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004702 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004703 ConfigureEncoder(std::move(video_encoder_config),
4704 VideoStreamEncoder::BitrateAllocationCallbackType::
4705 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004706
4707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004709
4710 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4711 WaitForEncodedFrame(CurrentTimeMs());
4712 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4713 VideoLayersAllocation last_layer_allocation =
4714 sink_.GetLastVideoLayersAllocation();
4715
4716 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4717 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4718 .target_bitrate_per_temporal_layer,
4719 SizeIs(2));
4720 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4721 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4722 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4723 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4724 .target_bitrate_per_temporal_layer,
4725 SizeIs(2));
4726 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4727 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4728 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4729
4730 // Since full SVC is used, expect the top layer to utilize the full target
4731 // rate.
4732 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4733 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004734 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004735 video_stream_encoder_->Stop();
4736}
4737
4738TEST_F(VideoStreamEncoderTest,
4739 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4740 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4741 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742 VideoEncoderConfig video_encoder_config;
4743 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4744 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004745 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004746 video_encoder_config.content_type =
4747 VideoEncoderConfig::ContentType::kRealtimeVideo;
4748 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4749 vp9_settings.numberOfSpatialLayers = 2;
4750 vp9_settings.numberOfTemporalLayers = 2;
4751 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4752 vp9_settings.automaticResizeOn = false;
4753 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004754 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004756 ConfigureEncoder(std::move(video_encoder_config),
4757 VideoStreamEncoder::BitrateAllocationCallbackType::
4758 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004759
4760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004761 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004762
4763 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4764 WaitForEncodedFrame(CurrentTimeMs());
4765 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4766 VideoLayersAllocation last_layer_allocation =
4767 sink_.GetLastVideoLayersAllocation();
4768
4769 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4770 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4771 .target_bitrate_per_temporal_layer,
4772 SizeIs(1));
4773 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4774 .target_bitrate_per_temporal_layer,
4775 SizeIs(1));
4776 // Since full SVC is used, expect the top layer to utilize the full target
4777 // rate.
4778 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4779 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004780 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004781 video_stream_encoder_->Stop();
4782}
4783
4784TEST_F(VideoStreamEncoderTest,
4785 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4786 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4787 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004788 VideoEncoderConfig video_encoder_config;
4789 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4790 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004791 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004792 video_encoder_config.content_type =
4793 VideoEncoderConfig::ContentType::kRealtimeVideo;
4794 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4795 vp9_settings.numberOfSpatialLayers = 2;
4796 vp9_settings.numberOfTemporalLayers = 2;
4797 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4798 vp9_settings.automaticResizeOn = false;
4799 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004800 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004801 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004802 ConfigureEncoder(std::move(video_encoder_config),
4803 VideoStreamEncoder::BitrateAllocationCallbackType::
4804 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004805
4806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004807 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808
4809 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4810 WaitForEncodedFrame(CurrentTimeMs());
4811 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4812 VideoLayersAllocation last_layer_allocation =
4813 sink_.GetLastVideoLayersAllocation();
4814
4815 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4816 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4817 .target_bitrate_per_temporal_layer,
4818 SizeIs(2));
4819 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4820 .target_bitrate_per_temporal_layer,
4821 SizeIs(2));
4822 // Since KSVC is, spatial layers are independend except on key frames.
4823 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4824 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004825 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004826 video_stream_encoder_->Stop();
4827}
4828
4829TEST_F(VideoStreamEncoderTest,
4830 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4831 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4832 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4833 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004834 VideoEncoderConfig video_encoder_config;
4835 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4836 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004837 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004838 video_encoder_config.content_type =
4839 VideoEncoderConfig::ContentType::kRealtimeVideo;
4840 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4841 vp9_settings.numberOfSpatialLayers = 3;
4842 vp9_settings.numberOfTemporalLayers = 2;
4843 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4844 vp9_settings.automaticResizeOn = false;
4845 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004846 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004847 vp9_settings);
4848 // Simulcast layers are used for enabling/disabling streams.
4849 video_encoder_config.simulcast_layers.resize(3);
4850 video_encoder_config.simulcast_layers[0].active = false;
4851 video_encoder_config.simulcast_layers[1].active = true;
4852 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004853 ConfigureEncoder(std::move(video_encoder_config),
4854 VideoStreamEncoder::BitrateAllocationCallbackType::
4855 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004856
4857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004859
4860 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4861 WaitForEncodedFrame(CurrentTimeMs());
4862 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4863 VideoLayersAllocation last_layer_allocation =
4864 sink_.GetLastVideoLayersAllocation();
4865
4866 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4867 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4868 .target_bitrate_per_temporal_layer,
4869 SizeIs(2));
4870 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4871 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4872
4873 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4874 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4875 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4876 .target_bitrate_per_temporal_layer,
4877 SizeIs(2));
4878 // Since full SVC is used, expect the top layer to utilize the full target
4879 // rate.
4880 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4881 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004882 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004883 video_stream_encoder_->Stop();
4884}
4885
4886TEST_F(VideoStreamEncoderTest,
4887 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4888 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4889 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4890 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004891 VideoEncoderConfig video_encoder_config;
4892 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4893 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004894 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004895 video_encoder_config.content_type =
4896 VideoEncoderConfig::ContentType::kRealtimeVideo;
4897 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4898 vp9_settings.numberOfSpatialLayers = 3;
4899 vp9_settings.numberOfTemporalLayers = 2;
4900 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4901 vp9_settings.automaticResizeOn = false;
4902 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004903 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004904 vp9_settings);
4905 // Simulcast layers are used for enabling/disabling streams.
4906 video_encoder_config.simulcast_layers.resize(3);
4907 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004908 ConfigureEncoder(std::move(video_encoder_config),
4909 VideoStreamEncoder::BitrateAllocationCallbackType::
4910 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004911
4912 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004913 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004914
4915 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4916 WaitForEncodedFrame(CurrentTimeMs());
4917 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4918 VideoLayersAllocation last_layer_allocation =
4919 sink_.GetLastVideoLayersAllocation();
4920
4921 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4922 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4923 .target_bitrate_per_temporal_layer,
4924 SizeIs(2));
4925 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4926 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4927
4928 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4929 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4930 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4931 .target_bitrate_per_temporal_layer,
4932 SizeIs(2));
4933 video_stream_encoder_->Stop();
4934}
4935
4936TEST_F(VideoStreamEncoderTest,
4937 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4938 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4939 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4940 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004941 VideoEncoderConfig video_encoder_config;
4942 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4943 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004944 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004945 video_encoder_config.content_type =
4946 VideoEncoderConfig::ContentType::kRealtimeVideo;
4947 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4948 vp9_settings.numberOfSpatialLayers = 3;
4949 vp9_settings.numberOfTemporalLayers = 2;
4950 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4951 vp9_settings.automaticResizeOn = false;
4952 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004953 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004954 vp9_settings);
4955 // Simulcast layers are used for enabling/disabling streams.
4956 video_encoder_config.simulcast_layers.resize(3);
4957 video_encoder_config.simulcast_layers[0].active = false;
4958 video_encoder_config.simulcast_layers[1].active = false;
4959 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004960 ConfigureEncoder(std::move(video_encoder_config),
4961 VideoStreamEncoder::BitrateAllocationCallbackType::
4962 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004963
4964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004965 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004966
4967 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4968 WaitForEncodedFrame(CurrentTimeMs());
4969 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4970 VideoLayersAllocation last_layer_allocation =
4971 sink_.GetLastVideoLayersAllocation();
4972
4973 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4974 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4975 .target_bitrate_per_temporal_layer,
4976 SizeIs(2));
4977 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4978 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4979 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4980 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004981 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004982 video_stream_encoder_->Stop();
4983}
4984
4985TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4986 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004987 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004988 kVideoLayersAllocation);
4989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004990 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004991
4992 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4993 WaitForEncodedFrame(CurrentTimeMs());
4994 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4995 VideoLayersAllocation last_layer_allocation =
4996 sink_.GetLastVideoLayersAllocation();
4997
4998 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4999 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5000 .target_bitrate_per_temporal_layer,
5001 SizeIs(1));
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5003 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005004 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005005 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5006 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5007 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5008 video_stream_encoder_->Stop();
5009}
5010
5011TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005012 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5013 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005014 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005015 kVideoLayersAllocation);
5016
5017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005018 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005019
5020 video_source_.IncomingCapturedFrame(
5021 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5022 WaitForEncodedFrame(CurrentTimeMs());
5023 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5024 VideoLayersAllocation last_layer_allocation =
5025 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005026 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005027 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5028 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5029 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005030 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005031
5032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005033 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5034 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005035 video_source_.IncomingCapturedFrame(
5036 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5037 WaitForEncodedFrame(CurrentTimeMs());
5038
5039 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5040 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5041 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5042 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5043 .target_bitrate_per_temporal_layer[0],
5044 DataRate::Zero());
5045
5046 video_stream_encoder_->Stop();
5047}
5048
Per Kjellander4190ce92020-12-15 17:24:55 +01005049TEST_F(VideoStreamEncoderTest,
5050 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5051 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005052 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005053 kVideoLayersAllocation);
5054
5055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005056 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5057 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005058
5059 video_source_.IncomingCapturedFrame(
5060 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5061 WaitForEncodedFrame(CurrentTimeMs());
5062 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5063 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5064 SizeIs(2));
5065 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5066 codec_width_);
5067 EXPECT_EQ(
5068 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5069 codec_height_);
5070
5071 video_source_.IncomingCapturedFrame(
5072 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5073 WaitForEncodedFrame(CurrentTimeMs());
5074 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5075 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5076 SizeIs(2));
5077 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5078 codec_width_ / 2);
5079 EXPECT_EQ(
5080 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5081 codec_height_ / 2);
5082
5083 video_stream_encoder_->Stop();
5084}
5085
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005086TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5087 // 2 TLs configured, temporal layers supported by encoder.
5088 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005089 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005090 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005091 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005092 fake_encoder_.SetTemporalLayersSupported(0, true);
5093
5094 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005095 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005096 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005097 kNumTemporalLayers, /*temporal_id*/ 0,
5098 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005099 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005100 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005101 kNumTemporalLayers, /*temporal_id*/ 1,
5102 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005103 VideoBitrateAllocation expected_bitrate;
5104 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5105 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5106
5107 VerifyAllocatedBitrate(expected_bitrate);
5108 video_stream_encoder_->Stop();
5109}
5110
5111TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5112 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005113 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005114 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005115 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005116 fake_encoder_.SetTemporalLayersSupported(0, false);
5117
5118 // Temporal layers not supported by the encoder.
5119 // Total bitrate should be at ti:0.
5120 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005121 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005122
5123 VerifyAllocatedBitrate(expected_bitrate);
5124 video_stream_encoder_->Stop();
5125}
5126
5127TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005128 webrtc::test::ScopedFieldTrials field_trials(
5129 "WebRTC-Video-QualityScalerSettings/"
5130 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5131 // Reset encoder for field trials to take effect.
5132 ConfigureEncoder(video_encoder_config_.Copy());
5133
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005134 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005135 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005136 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005137 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005138 fake_encoder_.SetTemporalLayersSupported(0, true);
5139 fake_encoder_.SetTemporalLayersSupported(1, false);
5140
5141 const int kS0Bps = 150000;
5142 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005143 kS0Bps *
5144 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5145 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005146 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005147 kS0Bps *
5148 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5149 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005150 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005151 // Temporal layers not supported by si:1.
5152 VideoBitrateAllocation expected_bitrate;
5153 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5154 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5155 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5156
5157 VerifyAllocatedBitrate(expected_bitrate);
5158 video_stream_encoder_->Stop();
5159}
5160
Niels Möller7dc26b72017-12-06 10:27:48 +01005161TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5162 const int kFrameWidth = 1280;
5163 const int kFrameHeight = 720;
5164 const int kFramerate = 24;
5165
Henrik Boström381d1092020-05-12 18:49:07 +02005166 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005167 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005168 test::FrameForwarder source;
5169 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005170 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005171
5172 // Insert a single frame, triggering initial configuration.
5173 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5174 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5175
5176 EXPECT_EQ(
5177 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5178 kDefaultFramerate);
5179
5180 // Trigger reconfigure encoder (without resetting the entire instance).
5181 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005182 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5183 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005184 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005185 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005186 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005187 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5188
5189 // Detector should be updated with fps limit from codec config.
5190 EXPECT_EQ(
5191 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5192 kFramerate);
5193
5194 // Trigger overuse, max framerate should be reduced.
5195 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5196 stats.input_frame_rate = kFramerate;
5197 stats_proxy_->SetMockStats(stats);
5198 video_stream_encoder_->TriggerCpuOveruse();
5199 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5200 int adapted_framerate =
5201 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5202 EXPECT_LT(adapted_framerate, kFramerate);
5203
5204 // Trigger underuse, max framerate should go back to codec configured fps.
5205 // Set extra low fps, to make sure it's actually reset, not just incremented.
5206 stats = stats_proxy_->GetStats();
5207 stats.input_frame_rate = adapted_framerate / 2;
5208 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005209 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005210 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5211 EXPECT_EQ(
5212 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5213 kFramerate);
5214
5215 video_stream_encoder_->Stop();
5216}
5217
5218TEST_F(VideoStreamEncoderTest,
5219 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5220 const int kFrameWidth = 1280;
5221 const int kFrameHeight = 720;
5222 const int kLowFramerate = 15;
5223 const int kHighFramerate = 25;
5224
Henrik Boström381d1092020-05-12 18:49:07 +02005225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005226 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005227 test::FrameForwarder source;
5228 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005229 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005230
5231 // Trigger initial configuration.
5232 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005233 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5234 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005235 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005236 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005237 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005238 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005239 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5240
5241 EXPECT_EQ(
5242 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5243 kLowFramerate);
5244
5245 // Trigger overuse, max framerate should be reduced.
5246 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5247 stats.input_frame_rate = kLowFramerate;
5248 stats_proxy_->SetMockStats(stats);
5249 video_stream_encoder_->TriggerCpuOveruse();
5250 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5251 int adapted_framerate =
5252 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5253 EXPECT_LT(adapted_framerate, kLowFramerate);
5254
5255 // Reconfigure the encoder with a new (higher max framerate), max fps should
5256 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005257 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005258 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5259 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005260 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005261 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5262
5263 EXPECT_EQ(
5264 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5265 adapted_framerate);
5266
5267 // Trigger underuse, max framerate should go back to codec configured fps.
5268 stats = stats_proxy_->GetStats();
5269 stats.input_frame_rate = adapted_framerate;
5270 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005271 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005272 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5273 EXPECT_EQ(
5274 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5275 kHighFramerate);
5276
5277 video_stream_encoder_->Stop();
5278}
5279
mflodmancc3d4422017-08-03 08:27:51 -07005280TEST_F(VideoStreamEncoderTest,
5281 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005282 const int kFrameWidth = 1280;
5283 const int kFrameHeight = 720;
5284 const int kFramerate = 24;
5285
Henrik Boström381d1092020-05-12 18:49:07 +02005286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005288 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005289 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005290 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005291
5292 // Trigger initial configuration.
5293 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005294 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5295 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005296 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005297 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005298 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005299 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005300 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005301
Niels Möller7dc26b72017-12-06 10:27:48 +01005302 EXPECT_EQ(
5303 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5304 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005305
5306 // Trigger overuse, max framerate should be reduced.
5307 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5308 stats.input_frame_rate = kFramerate;
5309 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005310 video_stream_encoder_->TriggerCpuOveruse();
5311 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005312 int adapted_framerate =
5313 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005314 EXPECT_LT(adapted_framerate, kFramerate);
5315
5316 // Change degradation preference to not enable framerate scaling. Target
5317 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005318 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005319 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005320 EXPECT_EQ(
5321 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5322 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005323
mflodmancc3d4422017-08-03 08:27:51 -07005324 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005325}
5326
mflodmancc3d4422017-08-03 08:27:51 -07005327TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005328 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005330 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5331 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5332 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005333 const int kWidth = 640;
5334 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005335
asaperssonfab67072017-04-04 05:51:49 -07005336 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005337
5338 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005339 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005340
5341 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005342 EXPECT_TRUE_WAIT(
5343 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005344
sprangc5d62e22017-04-02 23:53:04 -07005345 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005346
asaperssonfab67072017-04-04 05:51:49 -07005347 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005348 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005349 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005350
5351 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005352 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005353
Henrik Boström2671dac2020-05-19 16:29:09 +02005354 EXPECT_TRUE_WAIT(
5355 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005356
mflodmancc3d4422017-08-03 08:27:51 -07005357 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005358}
5359
mflodmancc3d4422017-08-03 08:27:51 -07005360TEST_F(VideoStreamEncoderTest,
5361 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005362 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005363 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005364 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5365 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5366 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005367 const int kWidth = 640;
5368 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005369
5370 // We expect the n initial frames to get dropped.
5371 int i;
5372 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005373 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005374 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005375 }
5376 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005377 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005378 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005379
5380 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005381 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005382
mflodmancc3d4422017-08-03 08:27:51 -07005383 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005384}
5385
mflodmancc3d4422017-08-03 08:27:51 -07005386TEST_F(VideoStreamEncoderTest,
5387 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005388 const int kWidth = 640;
5389 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005391 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005392
5393 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005394 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005395 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005396
asaperssonfab67072017-04-04 05:51:49 -07005397 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005398 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005399 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005400
mflodmancc3d4422017-08-03 08:27:51 -07005401 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005402}
5403
mflodmancc3d4422017-08-03 08:27:51 -07005404TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005405 const int kWidth = 640;
5406 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005407 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005408
5409 VideoEncoderConfig video_encoder_config;
5410 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5411 // Make format different, to force recreation of encoder.
5412 video_encoder_config.video_format.parameters["foo"] = "foo";
5413 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005414 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005416 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005417
kthelgasonb83797b2017-02-14 11:57:25 -08005418 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005419 video_stream_encoder_->SetSource(&video_source_,
5420 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005421
asaperssonfab67072017-04-04 05:51:49 -07005422 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005423 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005424 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005425
mflodmancc3d4422017-08-03 08:27:51 -07005426 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005427 fake_encoder_.SetQualityScaling(true);
5428}
5429
Åsa Persson139f4dc2019-08-02 09:29:58 +02005430TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5431 webrtc::test::ScopedFieldTrials field_trials(
5432 "WebRTC-Video-QualityScalerSettings/"
5433 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5434 // Reset encoder for field trials to take effect.
5435 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005436 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5437 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005438 const int kWidth = 640;
5439 const int kHeight = 360;
5440
Henrik Boström381d1092020-05-12 18:49:07 +02005441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005442 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005443 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5444 // Frame should not be dropped.
5445 WaitForEncodedFrame(1);
5446
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005448 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5449 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5450 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005451 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5452 // Frame should not be dropped.
5453 WaitForEncodedFrame(2);
5454
Henrik Boström381d1092020-05-12 18:49:07 +02005455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005456 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5457 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5458 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005459 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5460 // Expect to drop this frame, the wait should time out.
5461 ExpectDroppedFrame();
5462
5463 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005464 EXPECT_TRUE_WAIT(
5465 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005466 video_stream_encoder_->Stop();
5467}
5468
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005469TEST_F(VideoStreamEncoderTest,
5470 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5471 webrtc::test::ScopedFieldTrials field_trials(
5472 "WebRTC-Video-QualityScalerSettings/"
5473 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5474 fake_encoder_.SetQualityScaling(false);
5475 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005476 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5477 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005478 const int kWidth = 640;
5479 const int kHeight = 360;
5480
5481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005482 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005483 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5484 // Frame should not be dropped.
5485 WaitForEncodedFrame(1);
5486
5487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5488 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5489 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5490 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5491 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5492 // Frame should not be dropped.
5493 WaitForEncodedFrame(2);
5494
5495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5496 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5497 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5498 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5499 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5500 // Not dropped since quality scaling is disabled.
5501 WaitForEncodedFrame(3);
5502
5503 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005504 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005505 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5506
5507 video_stream_encoder_->Stop();
5508}
5509
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005510TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005511 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005512 // Set simulcast.
5513 ResetEncoder("VP8", 3, 1, 1, false);
5514 fake_encoder_.SetQualityScaling(true);
5515 const int kWidth = 1280;
5516 const int kHeight = 720;
5517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005518 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005519 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5520 // Frame should not be dropped.
5521 WaitForEncodedFrame(1);
5522
5523 // Trigger QVGA "singlecast"
5524 // Update the config.
5525 VideoEncoderConfig video_encoder_config;
5526 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5527 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005528 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005529 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005530 "VP8", /*max qp*/ 56, /*screencast*/ false,
5531 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005532 for (auto& layer : video_encoder_config.simulcast_layers) {
5533 layer.num_temporal_layers = 1;
5534 layer.max_framerate = kDefaultFramerate;
5535 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005536 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005537 video_encoder_config.content_type =
5538 VideoEncoderConfig::ContentType::kRealtimeVideo;
5539
5540 video_encoder_config.simulcast_layers[0].active = true;
5541 video_encoder_config.simulcast_layers[1].active = false;
5542 video_encoder_config.simulcast_layers[2].active = false;
5543
5544 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5545 kMaxPayloadLength);
5546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5547
5548 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5549 // Frame should not be dropped.
5550 WaitForEncodedFrame(2);
5551
5552 // Trigger HD "singlecast"
5553 video_encoder_config.simulcast_layers[0].active = false;
5554 video_encoder_config.simulcast_layers[1].active = false;
5555 video_encoder_config.simulcast_layers[2].active = true;
5556
5557 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5558 kMaxPayloadLength);
5559 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5560
5561 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5562 // Frame should be dropped because of initial frame drop.
5563 ExpectDroppedFrame();
5564
5565 // Expect the sink_wants to specify a scaled frame.
5566 EXPECT_TRUE_WAIT(
5567 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5568 video_stream_encoder_->Stop();
5569}
5570
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005571TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005572 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005573 // Set simulcast.
5574 ResetEncoder("VP9", 1, 1, 3, false);
5575 fake_encoder_.SetQualityScaling(true);
5576 const int kWidth = 1280;
5577 const int kHeight = 720;
5578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005579 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005580 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5581 // Frame should not be dropped.
5582 WaitForEncodedFrame(1);
5583
5584 // Trigger QVGA "singlecast"
5585 // Update the config.
5586 VideoEncoderConfig video_encoder_config;
5587 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5588 &video_encoder_config);
5589 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5590 vp9_settings.numberOfSpatialLayers = 3;
5591 // Since only one layer is active - automatic resize should be enabled.
5592 vp9_settings.automaticResizeOn = true;
5593 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005594 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005595 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005596 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005597 video_encoder_config.content_type =
5598 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005599 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005600 // which SVC layers are active.
5601 video_encoder_config.simulcast_layers.resize(3);
5602
5603 video_encoder_config.simulcast_layers[0].active = true;
5604 video_encoder_config.simulcast_layers[1].active = false;
5605 video_encoder_config.simulcast_layers[2].active = false;
5606
5607 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5608 kMaxPayloadLength);
5609 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5610
5611 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5612 // Frame should not be dropped.
5613 WaitForEncodedFrame(2);
5614
5615 // Trigger HD "singlecast"
5616 video_encoder_config.simulcast_layers[0].active = false;
5617 video_encoder_config.simulcast_layers[1].active = false;
5618 video_encoder_config.simulcast_layers[2].active = true;
5619
5620 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5621 kMaxPayloadLength);
5622 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5623
5624 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5625 // Frame should be dropped because of initial frame drop.
5626 ExpectDroppedFrame();
5627
5628 // Expect the sink_wants to specify a scaled frame.
5629 EXPECT_TRUE_WAIT(
5630 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5631 video_stream_encoder_->Stop();
5632}
5633
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005634TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005635 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5636 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5637 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5638 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5639 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5640 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5641 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5642 fake_encoder_.SetResolutionBitrateLimits(
5643 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5644
5645 VideoEncoderConfig video_encoder_config;
5646 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5647 &video_encoder_config);
5648 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5649 vp9_settings.numberOfSpatialLayers = 3;
5650 // Since only one layer is active - automatic resize should be enabled.
5651 vp9_settings.automaticResizeOn = true;
5652 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005653 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005654 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005655 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005656 video_encoder_config.content_type =
5657 VideoEncoderConfig::ContentType::kRealtimeVideo;
5658 // Simulcast layers are used to indicate which spatial layers are active.
5659 video_encoder_config.simulcast_layers.resize(3);
5660 video_encoder_config.simulcast_layers[0].active = false;
5661 video_encoder_config.simulcast_layers[1].active = true;
5662 video_encoder_config.simulcast_layers[2].active = false;
5663
5664 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5665 kMaxPayloadLength);
5666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5667
5668 // The encoder bitrate limits for 360p should be used.
5669 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5670 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005671 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5672 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5673 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5674 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5675 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5676 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005677 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005678 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005679 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005680 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005681
5682 // The encoder bitrate limits for 270p should be used.
5683 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5684 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005685 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5686 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5687 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5688 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5689 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5690 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005691 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005692 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005693 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005694 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005695
5696 video_stream_encoder_->Stop();
5697}
5698
5699TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005700 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5701 VideoEncoderConfig video_encoder_config;
5702 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5703 &video_encoder_config);
5704 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5705 vp9_settings.numberOfSpatialLayers = 3;
5706 // Since only one layer is active - automatic resize should be enabled.
5707 vp9_settings.automaticResizeOn = true;
5708 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005709 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005710 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005711 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005712 video_encoder_config.content_type =
5713 VideoEncoderConfig::ContentType::kRealtimeVideo;
5714 // Simulcast layers are used to indicate which spatial layers are active.
5715 video_encoder_config.simulcast_layers.resize(3);
5716 video_encoder_config.simulcast_layers[0].active = false;
5717 video_encoder_config.simulcast_layers[1].active = true;
5718 video_encoder_config.simulcast_layers[2].active = false;
5719
5720 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5721 kMaxPayloadLength);
5722 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5723
5724 // The default bitrate limits for 360p should be used.
5725 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005726 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5727 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005728 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5729 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005730 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5731 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5732 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5733 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5734 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5735 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005736 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005737 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005738 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005739 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005740
5741 // The default bitrate limits for 270p should be used.
5742 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005743 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5744 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005745 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5746 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005747 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5748 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5749 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5750 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5751 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5752 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005753 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005754 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005755 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005756 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005757
5758 video_stream_encoder_->Stop();
5759}
5760
5761TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5762 webrtc::test::ScopedFieldTrials field_trials(
5763 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5764 VideoEncoderConfig video_encoder_config;
5765 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5766 &video_encoder_config);
5767 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5768 vp9_settings.numberOfSpatialLayers = 3;
5769 // Since only one layer is active - automatic resize should be enabled.
5770 vp9_settings.automaticResizeOn = true;
5771 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005772 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005773 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005774 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005775 video_encoder_config.content_type =
5776 VideoEncoderConfig::ContentType::kRealtimeVideo;
5777 // Simulcast layers are used to indicate which spatial layers are active.
5778 video_encoder_config.simulcast_layers.resize(3);
5779 video_encoder_config.simulcast_layers[0].active = false;
5780 video_encoder_config.simulcast_layers[1].active = true;
5781 video_encoder_config.simulcast_layers[2].active = false;
5782
5783 // Reset encoder for field trials to take effect.
5784 ConfigureEncoder(video_encoder_config.Copy());
5785
5786 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5787 kMaxPayloadLength);
5788 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5789
5790 // The default bitrate limits for 360p should not be used.
5791 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005792 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5793 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005794 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5795 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005796 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5797 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5798 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5799 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5800 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5801 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005802 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005804
5805 video_stream_encoder_->Stop();
5806}
5807
5808TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5809 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5810 /*num_spatial_layers=*/1, /*screenshare=*/false);
5811
5812 // The default singlecast bitrate limits for 720p should not be used.
5813 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005814 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5815 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005816 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5817 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005818 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5819 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5820 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5821 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5822 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5823 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005824 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005825 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005826
5827 video_stream_encoder_->Stop();
5828}
5829
5830TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005831 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5832 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5833 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5834 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5835 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5836 fake_encoder_.SetResolutionBitrateLimits(
5837 {kEncoderLimits180p, kEncoderLimits720p});
5838
5839 VideoEncoderConfig video_encoder_config;
5840 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5841 &video_encoder_config);
5842 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5843 vp9_settings.numberOfSpatialLayers = 3;
5844 // Since only one layer is active - automatic resize should be enabled.
5845 vp9_settings.automaticResizeOn = true;
5846 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005847 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005848 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005849 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005850 video_encoder_config.content_type =
5851 VideoEncoderConfig::ContentType::kRealtimeVideo;
5852 // Simulcast layers are used to indicate which spatial layers are active.
5853 video_encoder_config.simulcast_layers.resize(3);
5854 video_encoder_config.simulcast_layers[0].active = true;
5855 video_encoder_config.simulcast_layers[1].active = false;
5856 video_encoder_config.simulcast_layers[2].active = false;
5857
5858 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5859 kMaxPayloadLength);
5860 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5861
5862 // Limits not applied on lowest stream, limits for 180p should not be used.
5863 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5864 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005865 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5866 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5867 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5868 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5869 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5870 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005871 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005872 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005873 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005874 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005875
5876 video_stream_encoder_->Stop();
5877}
5878
5879TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005880 InitialFrameDropActivatesWhenResolutionIncreases) {
5881 const int kWidth = 640;
5882 const int kHeight = 360;
5883
5884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005886 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5887 // Frame should not be dropped.
5888 WaitForEncodedFrame(1);
5889
5890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005891 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005892 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5893 // Frame should not be dropped, bitrate not too low for frame.
5894 WaitForEncodedFrame(2);
5895
5896 // Incoming resolution increases.
5897 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5898 // Expect to drop this frame, bitrate too low for frame.
5899 ExpectDroppedFrame();
5900
5901 // Expect the sink_wants to specify a scaled frame.
5902 EXPECT_TRUE_WAIT(
5903 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5904 video_stream_encoder_->Stop();
5905}
5906
5907TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5908 const int kWidth = 640;
5909 const int kHeight = 360;
5910 // So that quality scaling doesn't happen by itself.
5911 fake_encoder_.SetQp(kQpHigh);
5912
5913 AdaptingFrameForwarder source(&time_controller_);
5914 source.set_adaptation_enabled(true);
5915 video_stream_encoder_->SetSource(
5916 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5917
5918 int timestamp = 1;
5919
5920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005921 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005922 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5923 WaitForEncodedFrame(timestamp);
5924 timestamp += 9000;
5925 // Long pause to disable all first BWE drop logic.
5926 AdvanceTime(TimeDelta::Millis(1000));
5927
5928 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005929 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005930 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5931 // Not dropped frame, as initial frame drop is disabled by now.
5932 WaitForEncodedFrame(timestamp);
5933 timestamp += 9000;
5934 AdvanceTime(TimeDelta::Millis(100));
5935
5936 // Quality adaptation down.
5937 video_stream_encoder_->TriggerQualityLow();
5938
5939 // Adaptation has an effect.
5940 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5941 5000);
5942
5943 // Frame isn't dropped as initial frame dropper is disabled.
5944 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5945 WaitForEncodedFrame(timestamp);
5946 timestamp += 9000;
5947 AdvanceTime(TimeDelta::Millis(100));
5948
5949 // Quality adaptation up.
5950 video_stream_encoder_->TriggerQualityHigh();
5951
5952 // Adaptation has an effect.
5953 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5954 5000);
5955
5956 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5957 // Frame should not be dropped, as initial framedropper is off.
5958 WaitForEncodedFrame(timestamp);
5959
5960 video_stream_encoder_->Stop();
5961}
5962
Åsa Persson7f354f82021-02-04 15:52:15 +01005963TEST_F(VideoStreamEncoderTest,
5964 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5965 const int kMinStartBps360p = 222000;
5966 fake_encoder_.SetResolutionBitrateLimits(
5967 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5968 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5969 800000)});
5970
5971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5972 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5973 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5974 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5975 0, 0, 0);
5976 // Frame should not be dropped, bitrate not too low for frame.
5977 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5978 WaitForEncodedFrame(1);
5979
5980 // Incoming resolution increases, initial frame drop activates.
5981 // Frame should be dropped, link allocation too low for frame.
5982 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5983 ExpectDroppedFrame();
5984
5985 // Expect sink_wants to specify a scaled frame.
5986 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5987 5000);
5988 video_stream_encoder_->Stop();
5989}
5990
5991TEST_F(VideoStreamEncoderTest,
5992 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5993 const int kMinStartBps360p = 222000;
5994 fake_encoder_.SetResolutionBitrateLimits(
5995 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5996 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5997 800000)});
5998
5999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6000 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6001 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6002 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6003 0, 0, 0);
6004 // Frame should not be dropped, bitrate not too low for frame.
6005 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6006 WaitForEncodedFrame(1);
6007
6008 // Incoming resolution increases, initial frame drop activates.
6009 // Frame should be dropped, link allocation not too low for frame.
6010 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6011 WaitForEncodedFrame(2);
6012
6013 video_stream_encoder_->Stop();
6014}
6015
Åsa Perssone644a032019-11-08 15:56:00 +01006016TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6017 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006018 "WebRTC-Video-QualityRampupSettings/"
6019 "min_pixels:921600,min_duration_ms:2000/");
6020
6021 const int kWidth = 1280;
6022 const int kHeight = 720;
6023 const int kFps = 10;
6024 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006025
6026 // Reset encoder for field trials to take effect.
6027 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006028 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006029 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006030 ConfigureEncoder(std::move(config));
6031 fake_encoder_.SetQp(kQpLow);
6032
6033 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006034 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006035 source.set_adaptation_enabled(true);
6036 video_stream_encoder_->SetSource(&source,
6037 DegradationPreference::MAINTAIN_FRAMERATE);
6038
6039 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006040 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006041 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006042 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006043
6044 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006045 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006046 int64_t timestamp_ms = kFrameIntervalMs;
6047 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6048 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006049 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6050 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006051
6052 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6054 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006055
Artem Titovab30d722021-07-27 16:22:11 +02006056 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006057 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006058 for (size_t i = 1; i <= 10; i++) {
6059 timestamp_ms += kFrameIntervalMs;
6060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6061 WaitForEncodedFrame(timestamp_ms);
6062 }
Åsa Persson06defc42021-09-10 15:28:48 +02006063
6064 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6065 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6066 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6067 timestamp_ms += kFrameIntervalMs;
6068 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6069 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006070 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6071 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6072
Åsa Persson06defc42021-09-10 15:28:48 +02006073 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006074 timestamp_ms += kFrameIntervalMs;
6075 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6076 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006077 // The ramp-up code involves the adaptation queue, give it time to execute.
6078 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006079 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006080 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006081
6082 // Frame should not be adapted.
6083 timestamp_ms += kFrameIntervalMs;
6084 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6085 WaitForEncodedFrame(kWidth, kHeight);
6086 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6087
6088 video_stream_encoder_->Stop();
6089}
6090
mflodmancc3d4422017-08-03 08:27:51 -07006091TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006092 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006093 webrtc::test::ScopedFieldTrials field_trials(
6094 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006095 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006096 source.set_adaptation_enabled(true);
6097 video_stream_encoder_->SetSource(&source,
6098 DegradationPreference::MAINTAIN_FRAMERATE);
6099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006100 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006101 fake_encoder_.SetQp(kQpHigh + 1);
6102 const int kWidth = 1280;
6103 const int kHeight = 720;
6104 const int64_t kFrameIntervalMs = 100;
6105 int64_t timestamp_ms = kFrameIntervalMs;
6106 for (size_t i = 1; i <= 100; i++) {
6107 timestamp_ms += kFrameIntervalMs;
6108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6109 WaitForEncodedFrame(timestamp_ms);
6110 }
6111 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6112 // for the first time.
6113 // TODO(eshr): We should avoid these waits by using threads with simulated
6114 // time.
6115 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6116 2000 * 2.5 * 2);
6117 timestamp_ms += kFrameIntervalMs;
6118 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6119 WaitForEncodedFrame(timestamp_ms);
6120 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6121 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6122 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6123
6124 // Disable Quality scaling by turning off scaler on the encoder and
6125 // reconfiguring.
6126 fake_encoder_.SetQualityScaling(false);
6127 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6128 kMaxPayloadLength);
6129 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006130 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006131 // Since we turned off the quality scaler, the adaptations made by it are
6132 // removed.
6133 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6134 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6135
6136 video_stream_encoder_->Stop();
6137}
6138
6139TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006140 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6141 const int kTooSmallWidth = 10;
6142 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006143 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006144 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006145
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006146 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006147 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006148 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006149 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006150 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006151 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6152
6153 // Trigger adapt down, too small frame, expect no change.
6154 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006155 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006156 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006157 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006158 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6159 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6160
mflodmancc3d4422017-08-03 08:27:51 -07006161 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006162}
6163
mflodmancc3d4422017-08-03 08:27:51 -07006164TEST_F(VideoStreamEncoderTest,
6165 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006166 const int kTooSmallWidth = 10;
6167 const int kTooSmallHeight = 10;
6168 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006170 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006171
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006172 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006173 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006174 video_stream_encoder_->SetSource(&source,
6175 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006176 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006177 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6178 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6179
6180 // Trigger adapt down, expect limited framerate.
6181 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006182 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006183 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006184 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006185 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6186 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6187 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6188
6189 // Trigger adapt down, too small frame, expect no change.
6190 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006191 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006192 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006193 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6195 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6196 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6197
mflodmancc3d4422017-08-03 08:27:51 -07006198 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006199}
6200
mflodmancc3d4422017-08-03 08:27:51 -07006201TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006202 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006203 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006204 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006205 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006206 const int kFrameWidth = 1280;
6207 const int kFrameHeight = 720;
6208 video_source_.IncomingCapturedFrame(
6209 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006210 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006211 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006212}
6213
sprangb1ca0732017-02-01 08:38:12 -08006214// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006215TEST_F(VideoStreamEncoderTest,
6216 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006218 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006219
6220 const int kFrameWidth = 1280;
6221 const int kFrameHeight = 720;
6222 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006223 // requested by
6224 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006225 video_source_.set_adaptation_enabled(true);
6226
6227 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006228 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006229 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006230
6231 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006232 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006233 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006234 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006235 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006236
asaperssonfab67072017-04-04 05:51:49 -07006237 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006238 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006239 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006240 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006241 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006242
mflodmancc3d4422017-08-03 08:27:51 -07006243 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006244}
sprangfe627f32017-03-29 08:24:59 -07006245
mflodmancc3d4422017-08-03 08:27:51 -07006246TEST_F(VideoStreamEncoderTest,
6247 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006248 const int kFrameWidth = 1280;
6249 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006250
Henrik Boström381d1092020-05-12 18:49:07 +02006251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006252 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006253 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006254 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006255 video_source_.set_adaptation_enabled(true);
6256
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006257 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006258
6259 video_source_.IncomingCapturedFrame(
6260 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006261 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006262
6263 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006264 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006265
6266 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006267 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006268 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006269 video_source_.IncomingCapturedFrame(
6270 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006271 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006272 }
6273
6274 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006275 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006276 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006277 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006278 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006279 video_source_.IncomingCapturedFrame(
6280 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006281 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006282 ++num_frames_dropped;
6283 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006284 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006285 }
6286 }
6287
sprang4847ae62017-06-27 07:06:52 -07006288 // Add some slack to account for frames dropped by the frame dropper.
6289 const int kErrorMargin = 1;
6290 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006291 kErrorMargin);
6292
6293 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006294 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006295 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006296 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006297 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006298 video_source_.IncomingCapturedFrame(
6299 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006300 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006301 ++num_frames_dropped;
6302 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006303 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006304 }
6305 }
sprang4847ae62017-06-27 07:06:52 -07006306 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006307 kErrorMargin);
6308
6309 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006310 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006311 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006312 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006313 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006314 video_source_.IncomingCapturedFrame(
6315 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006316 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006317 ++num_frames_dropped;
6318 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006319 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006320 }
6321 }
sprang4847ae62017-06-27 07:06:52 -07006322 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006323 kErrorMargin);
6324
6325 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006326 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006327 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006328 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006329 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006330 video_source_.IncomingCapturedFrame(
6331 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006332 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006333 ++num_frames_dropped;
6334 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006335 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006336 }
6337 }
6338 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6339
mflodmancc3d4422017-08-03 08:27:51 -07006340 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006341}
6342
mflodmancc3d4422017-08-03 08:27:51 -07006343TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006344 const int kFramerateFps = 5;
6345 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006346 const int kFrameWidth = 1280;
6347 const int kFrameHeight = 720;
6348
sprang4847ae62017-06-27 07:06:52 -07006349 // Reconfigure encoder with two temporal layers and screensharing, which will
6350 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006351 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006352
Henrik Boström381d1092020-05-12 18:49:07 +02006353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006354 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006355 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006356 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006357 video_source_.set_adaptation_enabled(true);
6358
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006359 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006360
6361 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006362 rtc::VideoSinkWants last_wants;
6363 do {
6364 last_wants = video_source_.sink_wants();
6365
sprangc5d62e22017-04-02 23:53:04 -07006366 // Insert frames to get a new fps estimate...
6367 for (int j = 0; j < kFramerateFps; ++j) {
6368 video_source_.IncomingCapturedFrame(
6369 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006370 if (video_source_.last_sent_width()) {
6371 sink_.WaitForEncodedFrame(timestamp_ms);
6372 }
sprangc5d62e22017-04-02 23:53:04 -07006373 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006374 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006375 }
6376 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006377 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006378 } while (video_source_.sink_wants().max_framerate_fps <
6379 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006380
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006381 EXPECT_THAT(video_source_.sink_wants(),
6382 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006383
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006385}
asaperssonf7e294d2017-06-13 23:25:22 -07006386
mflodmancc3d4422017-08-03 08:27:51 -07006387TEST_F(VideoStreamEncoderTest,
6388 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006389 const int kWidth = 1280;
6390 const int kHeight = 720;
6391 const int64_t kFrameIntervalMs = 150;
6392 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006393 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006394 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006395
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006396 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006397 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006398 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006399 video_stream_encoder_->SetSource(&source,
6400 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006401 timestamp_ms += kFrameIntervalMs;
6402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006403 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006404 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006405 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6407 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6408
6409 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006410 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006411 timestamp_ms += kFrameIntervalMs;
6412 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006413 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006414 EXPECT_THAT(source.sink_wants(),
6415 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006416 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6417 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6419
6420 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006421 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006422 timestamp_ms += kFrameIntervalMs;
6423 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006424 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006425 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006426 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6428 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6429
6430 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006431 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006432 timestamp_ms += kFrameIntervalMs;
6433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006434 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006435 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006436 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6437 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6438 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6439
6440 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006441 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006442 timestamp_ms += kFrameIntervalMs;
6443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006444 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006445 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006446 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6448 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6449
6450 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006451 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006452 timestamp_ms += kFrameIntervalMs;
6453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006454 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006455 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006456 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6457 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6458 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6459
6460 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006461 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006462 timestamp_ms += kFrameIntervalMs;
6463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006464 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006465 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6467 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6468 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6469
6470 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006471 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006472 timestamp_ms += kFrameIntervalMs;
6473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006474 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006475 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006476 rtc::VideoSinkWants last_wants = source.sink_wants();
6477 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6479 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6480
6481 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006482 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006483 timestamp_ms += kFrameIntervalMs;
6484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006486 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6489 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6490
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006491 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006492 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006493 timestamp_ms += kFrameIntervalMs;
6494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006495 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006496 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6499 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6500
6501 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006502 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006503 timestamp_ms += kFrameIntervalMs;
6504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006505 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006506 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6509 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6510
6511 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006512 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006513 timestamp_ms += kFrameIntervalMs;
6514 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006515 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006516 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6519 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6520
6521 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006522 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006523 timestamp_ms += kFrameIntervalMs;
6524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006525 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006526 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6529 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6530
6531 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006532 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006533 timestamp_ms += kFrameIntervalMs;
6534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006536 EXPECT_THAT(source.sink_wants(), FpsMax());
6537 EXPECT_EQ(source.sink_wants().max_pixel_count,
6538 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6541 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6542
6543 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006544 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006545 timestamp_ms += kFrameIntervalMs;
6546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006547 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6551 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6552
Åsa Persson30ab0152019-08-27 12:22:33 +02006553 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006554 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006555 timestamp_ms += kFrameIntervalMs;
6556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006557 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006559 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6562 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6563
6564 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006565 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006566 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006567 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
mflodmancc3d4422017-08-03 08:27:51 -07006569 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006570}
6571
mflodmancc3d4422017-08-03 08:27:51 -07006572TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006573 const int kWidth = 1280;
6574 const int kHeight = 720;
6575 const int64_t kFrameIntervalMs = 150;
6576 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006578 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006579
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006580 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006581 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006582 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006583 video_stream_encoder_->SetSource(&source,
6584 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006585 timestamp_ms += kFrameIntervalMs;
6586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006587 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006588 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006589 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6590 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6591 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6592 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6593 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6595
6596 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006597 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006598 timestamp_ms += kFrameIntervalMs;
6599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006600 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006601 EXPECT_THAT(source.sink_wants(),
6602 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006603 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6604 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6605 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6606 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6607 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6608 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
6610 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6617 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6618 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6619 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6620 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6621 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6622
6623 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006624 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006625 timestamp_ms += kFrameIntervalMs;
6626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006627 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006628 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6631 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6632 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6633 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6634 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6635
Evan Shrubsole64469032020-06-11 10:45:29 +02006636 // Trigger cpu adapt up, expect no change since QP is most limited.
6637 {
6638 // Store current sink wants since we expect no change and if there is no
6639 // change then last_wants() is not updated.
6640 auto previous_sink_wants = source.sink_wants();
6641 video_stream_encoder_->TriggerCpuUnderuse();
6642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6644 WaitForEncodedFrame(timestamp_ms);
6645 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6646 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6647 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6648 }
6649
6650 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6651 video_stream_encoder_->TriggerQualityHigh();
6652 timestamp_ms += kFrameIntervalMs;
6653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6654 WaitForEncodedFrame(timestamp_ms);
6655 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6658 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6660 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6661 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6662
6663 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6664 // expect increased resolution (960x540@30fps).
6665 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006666 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006667 timestamp_ms += kFrameIntervalMs;
6668 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006669 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006670 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006671 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6672 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6673 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6675 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006676 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006677
Evan Shrubsole64469032020-06-11 10:45:29 +02006678 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6679 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006680 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006681 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006682 timestamp_ms += kFrameIntervalMs;
6683 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006684 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6688 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6689 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6690 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6691 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006692 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006693
6694 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006695 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006696 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006697 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006698 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006699
mflodmancc3d4422017-08-03 08:27:51 -07006700 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006701}
6702
mflodmancc3d4422017-08-03 08:27:51 -07006703TEST_F(VideoStreamEncoderTest,
6704 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006705 const int kWidth = 640;
6706 const int kHeight = 360;
6707 const int kFpsLimit = 15;
6708 const int64_t kFrameIntervalMs = 150;
6709 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006711 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006712
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006713 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006714 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006715 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006716 video_stream_encoder_->SetSource(&source,
6717 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006718 timestamp_ms += kFrameIntervalMs;
6719 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006720 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006721 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6724 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6726 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6727 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6728
6729 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006730 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006731 timestamp_ms += kFrameIntervalMs;
6732 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006733 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006734 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6737 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6738 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6739 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6740 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6741
6742 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006743 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006744 timestamp_ms += kFrameIntervalMs;
6745 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006746 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006747 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006748 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006750 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6751 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6752 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6753 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6754
Evan Shrubsole64469032020-06-11 10:45:29 +02006755 // Trigger cpu adapt up, expect no change because quality is most limited.
6756 {
6757 auto previous_sink_wants = source.sink_wants();
6758 // Store current sink wants since we expect no change ind if there is no
6759 // change then last__wants() is not updated.
6760 video_stream_encoder_->TriggerCpuUnderuse();
6761 timestamp_ms += kFrameIntervalMs;
6762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6763 WaitForEncodedFrame(timestamp_ms);
6764 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6765 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6766 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6767 }
6768
6769 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6770 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006771 timestamp_ms += kFrameIntervalMs;
6772 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006773 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006774 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006775 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6776 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6777 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006778 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6779 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6780 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006781
Evan Shrubsole64469032020-06-11 10:45:29 +02006782 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006783 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006784 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006785 timestamp_ms += kFrameIntervalMs;
6786 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006787 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006788 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006789 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6792 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6793 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006794 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006795
6796 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006797 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006799 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006800 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006801
mflodmancc3d4422017-08-03 08:27:51 -07006802 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006803}
6804
mflodmancc3d4422017-08-03 08:27:51 -07006805TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006806 const int kFrameWidth = 1920;
6807 const int kFrameHeight = 1080;
6808 // 3/4 of 1920.
6809 const int kAdaptedFrameWidth = 1440;
6810 // 3/4 of 1080 rounded down to multiple of 4.
6811 const int kAdaptedFrameHeight = 808;
6812 const int kFramerate = 24;
6813
Henrik Boström381d1092020-05-12 18:49:07 +02006814 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006815 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006816 // Trigger reconfigure encoder (without resetting the entire instance).
6817 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006818 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6819 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006820 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006821 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006822 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006823 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006824 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006825 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006826
6827 video_source_.set_adaptation_enabled(true);
6828
6829 video_source_.IncomingCapturedFrame(
6830 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006831 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006832
6833 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006834 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006835 video_source_.IncomingCapturedFrame(
6836 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006837 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006838
mflodmancc3d4422017-08-03 08:27:51 -07006839 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006840}
6841
mflodmancc3d4422017-08-03 08:27:51 -07006842TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006843 const int kFrameWidth = 1280;
6844 const int kFrameHeight = 720;
6845 const int kLowFps = 2;
6846 const int kHighFps = 30;
6847
Henrik Boström381d1092020-05-12 18:49:07 +02006848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006850
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006851 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006852 max_framerate_ = kLowFps;
6853
6854 // Insert 2 seconds of 2fps video.
6855 for (int i = 0; i < kLowFps * 2; ++i) {
6856 video_source_.IncomingCapturedFrame(
6857 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6858 WaitForEncodedFrame(timestamp_ms);
6859 timestamp_ms += 1000 / kLowFps;
6860 }
6861
6862 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006864 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006865 video_source_.IncomingCapturedFrame(
6866 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6867 WaitForEncodedFrame(timestamp_ms);
6868 timestamp_ms += 1000 / kLowFps;
6869
6870 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6871
6872 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006873 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006874 const int kFrameIntervalMs = 1000 / kHighFps;
6875 max_framerate_ = kHighFps;
6876 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6877 video_source_.IncomingCapturedFrame(
6878 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6879 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6880 // be dropped if the encoder hans't been updated with the new higher target
6881 // framerate yet, causing it to overshoot the target bitrate and then
6882 // suffering the wrath of the media optimizer.
6883 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6884 timestamp_ms += kFrameIntervalMs;
6885 }
6886
6887 // Don expect correct measurement just yet, but it should be higher than
6888 // before.
6889 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6890
mflodmancc3d4422017-08-03 08:27:51 -07006891 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006892}
6893
mflodmancc3d4422017-08-03 08:27:51 -07006894TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006895 const int kFrameWidth = 1280;
6896 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006897 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006898 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006899 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006900
Henrik Boström381d1092020-05-12 18:49:07 +02006901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006902 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006903 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006904
6905 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006906 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006907 video_source_.IncomingCapturedFrame(
6908 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6909 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006910 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006911
6912 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006913 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006914 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006915
6916 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006917 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006918 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006919
Per Kjellanderdcef6412020-10-07 15:09:05 +02006920 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006921 video_source_.IncomingCapturedFrame(
6922 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6923 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006924 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006925
mflodmancc3d4422017-08-03 08:27:51 -07006926 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006927}
ilnik6b826ef2017-06-16 06:53:48 -07006928
Niels Möller4db138e2018-04-19 09:04:13 +02006929TEST_F(VideoStreamEncoderTest,
6930 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6931 const int kFrameWidth = 1280;
6932 const int kFrameHeight = 720;
6933 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006934 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006935 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006936 video_source_.IncomingCapturedFrame(
6937 CreateFrame(1, kFrameWidth, kFrameHeight));
6938 WaitForEncodedFrame(1);
6939 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6940 .low_encode_usage_threshold_percent,
6941 default_options.low_encode_usage_threshold_percent);
6942 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6943 .high_encode_usage_threshold_percent,
6944 default_options.high_encode_usage_threshold_percent);
6945 video_stream_encoder_->Stop();
6946}
6947
6948TEST_F(VideoStreamEncoderTest,
6949 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6950 const int kFrameWidth = 1280;
6951 const int kFrameHeight = 720;
6952 CpuOveruseOptions hardware_options;
6953 hardware_options.low_encode_usage_threshold_percent = 150;
6954 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006955 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006956
Henrik Boström381d1092020-05-12 18:49:07 +02006957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006958 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006959 video_source_.IncomingCapturedFrame(
6960 CreateFrame(1, kFrameWidth, kFrameHeight));
6961 WaitForEncodedFrame(1);
6962 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6963 .low_encode_usage_threshold_percent,
6964 hardware_options.low_encode_usage_threshold_percent);
6965 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6966 .high_encode_usage_threshold_percent,
6967 hardware_options.high_encode_usage_threshold_percent);
6968 video_stream_encoder_->Stop();
6969}
6970
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006971TEST_F(VideoStreamEncoderTest,
6972 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6973 const int kFrameWidth = 1280;
6974 const int kFrameHeight = 720;
6975
6976 const CpuOveruseOptions default_options;
6977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006978 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006979 video_source_.IncomingCapturedFrame(
6980 CreateFrame(1, kFrameWidth, kFrameHeight));
6981 WaitForEncodedFrame(1);
6982 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6983 .low_encode_usage_threshold_percent,
6984 default_options.low_encode_usage_threshold_percent);
6985 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6986 .high_encode_usage_threshold_percent,
6987 default_options.high_encode_usage_threshold_percent);
6988
6989 CpuOveruseOptions hardware_options;
6990 hardware_options.low_encode_usage_threshold_percent = 150;
6991 hardware_options.high_encode_usage_threshold_percent = 200;
6992 fake_encoder_.SetIsHardwareAccelerated(true);
6993
6994 video_source_.IncomingCapturedFrame(
6995 CreateFrame(2, kFrameWidth, kFrameHeight));
6996 WaitForEncodedFrame(2);
6997
6998 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6999 .low_encode_usage_threshold_percent,
7000 hardware_options.low_encode_usage_threshold_percent);
7001 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7002 .high_encode_usage_threshold_percent,
7003 hardware_options.high_encode_usage_threshold_percent);
7004
7005 video_stream_encoder_->Stop();
7006}
7007
Niels Möller6bb5ab92019-01-11 11:11:10 +01007008TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7009 const int kFrameWidth = 320;
7010 const int kFrameHeight = 240;
7011 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007012 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007013 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7014
Henrik Boström381d1092020-05-12 18:49:07 +02007015 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007016 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007017
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007018 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007019 max_framerate_ = kFps;
7020
7021 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7022 fake_encoder_.SimulateOvershoot(1.0);
7023 int num_dropped = 0;
7024 for (int i = 0; i < kNumFramesInRun; ++i) {
7025 video_source_.IncomingCapturedFrame(
7026 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7027 // Wait up to two frame durations for a frame to arrive.
7028 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7029 ++num_dropped;
7030 }
7031 timestamp_ms += 1000 / kFps;
7032 }
7033
Erik Språnga8d48ab2019-02-08 14:17:40 +01007034 // Framerate should be measured to be near the expected target rate.
7035 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7036
7037 // Frame drops should be within 5% of expected 0%.
7038 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007039
7040 // Make encoder produce frames at double the expected bitrate during 3 seconds
7041 // of video, verify number of drops. Rate needs to be slightly changed in
7042 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007043 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007044 const RateControlSettings trials =
7045 RateControlSettings::ParseFromFieldTrials();
7046 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007047 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007048 // frame dropping since the adjuter will try to just lower the target
7049 // bitrate rather than drop frames. If network headroom can be used, it
7050 // doesn't push back as hard so we don't need quite as much overshoot.
7051 // These numbers are unfortunately a bit magical but there's not trivial
7052 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007053 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007054 }
7055 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007057 kTargetBitrate + DataRate::KilobitsPerSec(1),
7058 kTargetBitrate + DataRate::KilobitsPerSec(1),
7059 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007060 num_dropped = 0;
7061 for (int i = 0; i < kNumFramesInRun; ++i) {
7062 video_source_.IncomingCapturedFrame(
7063 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7064 // Wait up to two frame durations for a frame to arrive.
7065 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7066 ++num_dropped;
7067 }
7068 timestamp_ms += 1000 / kFps;
7069 }
7070
Henrik Boström381d1092020-05-12 18:49:07 +02007071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007072 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007073
7074 // Target framerate should be still be near the expected target, despite
7075 // the frame drops.
7076 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7077
7078 // Frame drops should be within 5% of expected 50%.
7079 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007080
7081 video_stream_encoder_->Stop();
7082}
7083
7084TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7085 const int kFrameWidth = 320;
7086 const int kFrameHeight = 240;
7087 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007088 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007089
7090 ASSERT_GT(max_framerate_, kActualInputFps);
7091
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007092 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007093 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007095 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007096
7097 // Insert 3 seconds of video, with an input fps lower than configured max.
7098 for (int i = 0; i < kActualInputFps * 3; ++i) {
7099 video_source_.IncomingCapturedFrame(
7100 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7101 // Wait up to two frame durations for a frame to arrive.
7102 WaitForEncodedFrame(timestamp_ms);
7103 timestamp_ms += 1000 / kActualInputFps;
7104 }
7105
7106 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7107
7108 video_stream_encoder_->Stop();
7109}
7110
Markus Handell9a478b52021-11-18 16:07:01 +01007111TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007112 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007113 test::FrameForwarder source;
7114 video_stream_encoder_->SetSource(&source,
7115 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007117 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007118
Markus Handell9a478b52021-11-18 16:07:01 +01007119 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007120 WaitForEncodedFrame(1);
7121 // On the very first frame full update should be forced.
7122 rect = fake_encoder_.GetLastUpdateRect();
7123 EXPECT_EQ(rect.offset_x, 0);
7124 EXPECT_EQ(rect.offset_y, 0);
7125 EXPECT_EQ(rect.height, codec_height_);
7126 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007127 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7128 // scheduled for processing during encoder queue processing of frame 2.
7129 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7130 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007131 WaitForEncodedFrame(3);
7132 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7133 rect = fake_encoder_.GetLastUpdateRect();
7134 EXPECT_EQ(rect.offset_x, 1);
7135 EXPECT_EQ(rect.offset_y, 0);
7136 EXPECT_EQ(rect.width, 10);
7137 EXPECT_EQ(rect.height, 1);
7138
Markus Handell9a478b52021-11-18 16:07:01 +01007139 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007140 WaitForEncodedFrame(4);
7141 // Previous frame was encoded, so no accumulation should happen.
7142 rect = fake_encoder_.GetLastUpdateRect();
7143 EXPECT_EQ(rect.offset_x, 0);
7144 EXPECT_EQ(rect.offset_y, 0);
7145 EXPECT_EQ(rect.width, 1);
7146 EXPECT_EQ(rect.height, 1);
7147
7148 video_stream_encoder_->Stop();
7149}
7150
Erik Språngd7329ca2019-02-21 21:19:53 +01007151TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007153 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007154
7155 // First frame is always keyframe.
7156 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7157 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007158 EXPECT_THAT(
7159 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007160 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007161
7162 // Insert delta frame.
7163 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7164 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007165 EXPECT_THAT(
7166 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007167 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007168
7169 // Request next frame be a key-frame.
7170 video_stream_encoder_->SendKeyFrame();
7171 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7172 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007173 EXPECT_THAT(
7174 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007175 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007176
7177 video_stream_encoder_->Stop();
7178}
7179
7180TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7181 // Setup simulcast with three streams.
7182 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007184 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7185 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007186 // Wait for all three layers before triggering event.
7187 sink_.SetNumExpectedLayers(3);
7188
7189 // First frame is always keyframe.
7190 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7191 WaitForEncodedFrame(1);
7192 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007193 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7194 VideoFrameType::kVideoFrameKey,
7195 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007196
7197 // Insert delta frame.
7198 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7199 WaitForEncodedFrame(2);
7200 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007201 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7202 VideoFrameType::kVideoFrameDelta,
7203 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007204
7205 // Request next frame be a key-frame.
7206 // Only first stream is configured to produce key-frame.
7207 video_stream_encoder_->SendKeyFrame();
7208 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7209 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007210
7211 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7212 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007213 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007214 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007215 VideoFrameType::kVideoFrameKey,
7216 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007217
7218 video_stream_encoder_->Stop();
7219}
7220
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007221TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007222 // SPS contains VUI with restrictions on the maximum number of reordered
7223 // pictures, there is no need to rewrite the bitstream to enable faster
7224 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007225 ResetEncoder("H264", 1, 1, 1, false);
7226
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007227 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007228 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007229 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007230
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007231 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007232 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007233
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007234 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7235 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007236
7237 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007238 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007239
7240 video_stream_encoder_->Stop();
7241}
7242
7243TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007244 // SPS does not contain VUI, the bitstream is will be rewritten with added
7245 // VUI with restrictions on the maximum number of reordered pictures to
7246 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007247 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7248 0x00, 0x00, 0x03, 0x03, 0xF4,
7249 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007250 ResetEncoder("H264", 1, 1, 1, false);
7251
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007252 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007253 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007254 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007255
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007256 fake_encoder_.SetEncodedImageData(
7257 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007258
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007259 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7260 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007261
7262 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007263 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007264
7265 video_stream_encoder_->Stop();
7266}
7267
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007268TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7269 const int kFrameWidth = 1280;
7270 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007271 const DataRate kTargetBitrate =
7272 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007273
Henrik Boström381d1092020-05-12 18:49:07 +02007274 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007275 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007276 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7277
7278 // Insert a first video frame. It should be dropped because of downscale in
7279 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007280 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007281 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7282 frame.set_rotation(kVideoRotation_270);
7283 video_source_.IncomingCapturedFrame(frame);
7284
7285 ExpectDroppedFrame();
7286
7287 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007288 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007289 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7290 frame.set_rotation(kVideoRotation_90);
7291 video_source_.IncomingCapturedFrame(frame);
7292
7293 WaitForEncodedFrame(timestamp_ms);
7294 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7295
7296 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007297 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007298 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7299 frame.set_rotation(kVideoRotation_180);
7300 video_source_.IncomingCapturedFrame(frame);
7301
7302 WaitForEncodedFrame(timestamp_ms);
7303 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7304
7305 video_stream_encoder_->Stop();
7306}
7307
Erik Språng5056af02019-09-02 15:53:11 +02007308TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7309 const int kFrameWidth = 320;
7310 const int kFrameHeight = 180;
7311
7312 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007314 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7315 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7316 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007317 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007318 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007319 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007320
7321 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007322 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007323 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7324 frame.set_rotation(kVideoRotation_270);
7325 video_source_.IncomingCapturedFrame(frame);
7326 WaitForEncodedFrame(timestamp_ms);
7327
7328 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007329 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007330 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7331 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007333 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007334 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007335 /*link_allocation=*/target_rate,
7336 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007337 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007338 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007339 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7340
7341 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7342 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7343 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007344 DataRate allocation_sum =
7345 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007346 EXPECT_EQ(min_rate, allocation_sum);
7347 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7348
7349 video_stream_encoder_->Stop();
7350}
7351
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007352TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007354 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007355 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007356 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007357 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7358 WaitForEncodedFrame(1);
7359
7360 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7361 ASSERT_TRUE(prev_rate_settings.has_value());
7362 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7363 kDefaultFramerate);
7364
7365 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7366 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7367 timestamp_ms += 1000 / kDefaultFramerate;
7368 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7369 WaitForEncodedFrame(timestamp_ms);
7370 }
7371 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7372 kDefaultFramerate);
7373 // Capture larger frame to trigger a reconfigure.
7374 codec_height_ *= 2;
7375 codec_width_ *= 2;
7376 timestamp_ms += 1000 / kDefaultFramerate;
7377 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7378 WaitForEncodedFrame(timestamp_ms);
7379
7380 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7381 auto current_rate_settings =
7382 fake_encoder_.GetAndResetLastRateControlSettings();
7383 // Ensure we have actually reconfigured twice
7384 // The rate settings should have been set again even though
7385 // they haven't changed.
7386 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007387 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007388
7389 video_stream_encoder_->Stop();
7390}
7391
philipeld9cc8c02019-09-16 14:53:40 +02007392struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007393 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7394 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7395 MOCK_METHOD(void,
7396 RequestEncoderSwitch,
7397 (const webrtc::SdpVideoFormat& format),
7398 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007399};
7400
philipel9b058032020-02-10 11:30:00 +01007401TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7402 constexpr int kDontCare = 100;
7403 StrictMock<MockEncoderSelector> encoder_selector;
7404 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7405 &fake_encoder_, &encoder_selector);
7406 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7407
7408 // Reset encoder for new configuration to take effect.
7409 ConfigureEncoder(video_encoder_config_.Copy());
7410
7411 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7412
7413 video_source_.IncomingCapturedFrame(
7414 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007415 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007416 video_stream_encoder_->Stop();
7417
7418 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7419 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007420 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7421 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007422 video_stream_encoder_.reset();
7423}
7424
7425TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7426 constexpr int kDontCare = 100;
7427
7428 NiceMock<MockEncoderSelector> encoder_selector;
7429 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7430 video_send_config_.encoder_settings.encoder_switch_request_callback =
7431 &switch_callback;
7432 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7433 &fake_encoder_, &encoder_selector);
7434 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7435
7436 // Reset encoder for new configuration to take effect.
7437 ConfigureEncoder(video_encoder_config_.Copy());
7438
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007439 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007440 .WillByDefault(Return(SdpVideoFormat("AV1")));
7441 EXPECT_CALL(switch_callback,
7442 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7443 Field(&SdpVideoFormat::name, "AV1"))));
7444
Henrik Boström381d1092020-05-12 18:49:07 +02007445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007446 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7447 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7448 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007449 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007450 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007451 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007452 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007453
7454 video_stream_encoder_->Stop();
7455}
7456
7457TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7458 constexpr int kSufficientBitrateToNotDrop = 1000;
7459 constexpr int kDontCare = 100;
7460
7461 NiceMock<MockVideoEncoder> video_encoder;
7462 NiceMock<MockEncoderSelector> encoder_selector;
7463 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7464 video_send_config_.encoder_settings.encoder_switch_request_callback =
7465 &switch_callback;
7466 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7467 &video_encoder, &encoder_selector);
7468 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7469
7470 // Reset encoder for new configuration to take effect.
7471 ConfigureEncoder(video_encoder_config_.Copy());
7472
7473 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7474 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7475 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007477 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7478 /*stable_target_bitrate=*/
7479 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7480 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007481 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007482 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007483 /*cwnd_reduce_ratio=*/0);
7484
7485 ON_CALL(video_encoder, Encode(_, _))
7486 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7487 ON_CALL(encoder_selector, OnEncoderBroken())
7488 .WillByDefault(Return(SdpVideoFormat("AV2")));
7489
7490 rtc::Event encode_attempted;
7491 EXPECT_CALL(switch_callback,
7492 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7493 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7494 EXPECT_EQ(format.name, "AV2");
7495 encode_attempted.Set();
7496 });
7497
7498 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7499 encode_attempted.Wait(3000);
7500
Markus Handell28c71802021-11-08 10:11:55 +01007501 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007502
philipel9b058032020-02-10 11:30:00 +01007503 video_stream_encoder_->Stop();
7504
7505 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7506 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007507 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7508 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007509 video_stream_encoder_.reset();
7510}
7511
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007512TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007513 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007514 const int kFrameWidth = 320;
7515 const int kFrameHeight = 180;
7516
7517 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007518 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007520 /*target_bitrate=*/rate,
7521 /*stable_target_bitrate=*/rate,
7522 /*link_allocation=*/rate,
7523 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007524 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007525 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007526
7527 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007528 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007529 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7530 frame.set_rotation(kVideoRotation_270);
7531 video_source_.IncomingCapturedFrame(frame);
7532 WaitForEncodedFrame(timestamp_ms);
7533 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7534
7535 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007536 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007538 /*target_bitrate=*/new_stable_rate,
7539 /*stable_target_bitrate=*/new_stable_rate,
7540 /*link_allocation=*/rate,
7541 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007542 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007543 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007544 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7545 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7546 video_stream_encoder_->Stop();
7547}
7548
7549TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007550 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007551 const int kFrameWidth = 320;
7552 const int kFrameHeight = 180;
7553
7554 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007555 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007556 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007557 /*target_bitrate=*/rate,
7558 /*stable_target_bitrate=*/rate,
7559 /*link_allocation=*/rate,
7560 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007561 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007562 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007563
7564 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007565 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007566 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7567 frame.set_rotation(kVideoRotation_270);
7568 video_source_.IncomingCapturedFrame(frame);
7569 WaitForEncodedFrame(timestamp_ms);
7570 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7571
7572 // Set a higher target rate without changing the link_allocation. Should not
7573 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007574 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007575 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007576 /*target_bitrate=*/rate,
7577 /*stable_target_bitrate=*/new_stable_rate,
7578 /*link_allocation=*/rate,
7579 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007580 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007581 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007582 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7583 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7584 video_stream_encoder_->Stop();
7585}
7586
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007587TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7588 test::ScopedFieldTrials field_trials(
7589 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7590 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7591 const int kFramerateFps = 30;
7592 const int kWidth = 1920;
7593 const int kHeight = 1080;
7594 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7595 // Works on screenshare mode.
7596 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7597 // We rely on the automatic resolution adaptation, but we handle framerate
7598 // adaptation manually by mocking the stats proxy.
7599 video_source_.set_adaptation_enabled(true);
7600
7601 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007603 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007604 video_stream_encoder_->SetSource(&video_source_,
7605 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007606 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007607
7608 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7609 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7610
7611 // Pass enough frames with the full update to trigger animation detection.
7612 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007613 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007614 frame.set_ntp_time_ms(timestamp_ms);
7615 frame.set_timestamp_us(timestamp_ms * 1000);
7616 video_source_.IncomingCapturedFrame(frame);
7617 WaitForEncodedFrame(timestamp_ms);
7618 }
7619
7620 // Resolution should be limited.
7621 rtc::VideoSinkWants expected;
7622 expected.max_framerate_fps = kFramerateFps;
7623 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007624 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007625
7626 // Pass one frame with no known update.
7627 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007628 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007629 frame.set_ntp_time_ms(timestamp_ms);
7630 frame.set_timestamp_us(timestamp_ms * 1000);
7631 frame.clear_update_rect();
7632
7633 video_source_.IncomingCapturedFrame(frame);
7634 WaitForEncodedFrame(timestamp_ms);
7635
7636 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007637 EXPECT_THAT(video_source_.sink_wants(),
7638 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007639
7640 video_stream_encoder_->Stop();
7641}
7642
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007643TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7644 const int kWidth = 720; // 540p adapted down.
7645 const int kHeight = 405;
7646 const int kNumFrames = 3;
7647 // Works on screenshare mode.
7648 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7649 /*num_spatial_layers=*/2, /*screenshare=*/true);
7650
7651 video_source_.set_adaptation_enabled(true);
7652
7653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007654 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007655
7656 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7657
7658 // Pass enough frames with the full update to trigger animation detection.
7659 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007660 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007661 frame.set_ntp_time_ms(timestamp_ms);
7662 frame.set_timestamp_us(timestamp_ms * 1000);
7663 video_source_.IncomingCapturedFrame(frame);
7664 WaitForEncodedFrame(timestamp_ms);
7665 }
7666
7667 video_stream_encoder_->Stop();
7668}
7669
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007670TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7671 const float downscale_factors[] = {4.0, 2.0, 1.0};
7672 const int number_layers =
7673 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7674 VideoEncoderConfig config;
7675 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7676 for (int i = 0; i < number_layers; ++i) {
7677 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7678 config.simulcast_layers[i].active = true;
7679 }
7680 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007681 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007682 "VP8", /*max qp*/ 56, /*screencast*/ false,
7683 /*screenshare enabled*/ false);
7684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007685 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7686 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007687
7688 // First initialization.
7689 // Encoder should be initialized. Next frame should be key frame.
7690 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7691 sink_.SetNumExpectedLayers(number_layers);
7692 int64_t timestamp_ms = kFrameIntervalMs;
7693 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7694 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007695 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007696 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7697 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7698 VideoFrameType::kVideoFrameKey,
7699 VideoFrameType::kVideoFrameKey}));
7700
7701 // Disable top layer.
7702 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7703 config.simulcast_layers[number_layers - 1].active = false;
7704 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7705 sink_.SetNumExpectedLayers(number_layers - 1);
7706 timestamp_ms += kFrameIntervalMs;
7707 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7708 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007709 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007710 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7711 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7712 VideoFrameType::kVideoFrameDelta,
7713 VideoFrameType::kVideoFrameDelta}));
7714
7715 // Re-enable top layer.
7716 // Encoder should be re-initialized. Next frame should be key frame.
7717 config.simulcast_layers[number_layers - 1].active = true;
7718 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7719 sink_.SetNumExpectedLayers(number_layers);
7720 timestamp_ms += kFrameIntervalMs;
7721 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7722 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007723 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007724 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7725 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7726 VideoFrameType::kVideoFrameKey,
7727 VideoFrameType::kVideoFrameKey}));
7728
7729 // Top layer max rate change.
7730 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7731 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7732 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7733 sink_.SetNumExpectedLayers(number_layers);
7734 timestamp_ms += kFrameIntervalMs;
7735 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7736 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007737 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007738 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7739 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7740 VideoFrameType::kVideoFrameDelta,
7741 VideoFrameType::kVideoFrameDelta}));
7742
7743 // Top layer resolution change.
7744 // Encoder should be re-initialized. Next frame should be key frame.
7745 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7746 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7747 sink_.SetNumExpectedLayers(number_layers);
7748 timestamp_ms += kFrameIntervalMs;
7749 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7750 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007751 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007752 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7753 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7754 VideoFrameType::kVideoFrameKey,
7755 VideoFrameType::kVideoFrameKey}));
7756 video_stream_encoder_->Stop();
7757}
7758
Henrik Boström1124ed12021-02-25 10:30:39 +01007759TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7760 const int kFrameWidth = 1280;
7761 const int kFrameHeight = 720;
7762
7763 SetUp();
7764 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007765 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007766
7767 // Capturing a frame should reconfigure the encoder and expose the encoder
7768 // resolution, which is the same as the input frame.
7769 int64_t timestamp_ms = kFrameIntervalMs;
7770 video_source_.IncomingCapturedFrame(
7771 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7772 WaitForEncodedFrame(timestamp_ms);
7773 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7774 EXPECT_THAT(video_source_.sink_wants().resolutions,
7775 ::testing::ElementsAreArray(
7776 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7777
7778 video_stream_encoder_->Stop();
7779}
7780
7781TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7782 // Pick downscale factors such that we never encode at full resolution - this
7783 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007784 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007785 // encoder should not ask for the frame resolution. This allows video frames
7786 // to have the appearence of one resolution but optimize its internal buffers
7787 // for what is actually encoded.
7788 const size_t kNumSimulcastLayers = 3u;
7789 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7790 const int kFrameWidth = 1280;
7791 const int kFrameHeight = 720;
7792 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7793 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7794 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7795 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7796 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7797 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7798
7799 VideoEncoderConfig config;
7800 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7801 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7802 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7803 config.simulcast_layers[i].active = true;
7804 }
7805 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007806 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007807 "VP8", /*max qp*/ 56, /*screencast*/ false,
7808 /*screenshare enabled*/ false);
7809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007810 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7811 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007812
7813 // Capture a frame with all layers active.
7814 int64_t timestamp_ms = kFrameIntervalMs;
7815 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7816 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7817 video_source_.IncomingCapturedFrame(
7818 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7819 WaitForEncodedFrame(timestamp_ms);
7820 // Expect encoded resolutions to match the expected simulcast layers.
7821 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7822 EXPECT_THAT(
7823 video_source_.sink_wants().resolutions,
7824 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7825
7826 // Capture a frame with one of the layers inactive.
7827 timestamp_ms += kFrameIntervalMs;
7828 config.simulcast_layers[2].active = false;
7829 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7830 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7831 video_source_.IncomingCapturedFrame(
7832 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7833 WaitForEncodedFrame(timestamp_ms);
7834
7835 // Expect encoded resolutions to match the expected simulcast layers.
7836 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7837 EXPECT_THAT(video_source_.sink_wants().resolutions,
7838 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7839
7840 // Capture a frame with all but one layer turned off.
7841 timestamp_ms += kFrameIntervalMs;
7842 config.simulcast_layers[1].active = false;
7843 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7844 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7845 video_source_.IncomingCapturedFrame(
7846 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7847 WaitForEncodedFrame(timestamp_ms);
7848
7849 // Expect encoded resolutions to match the expected simulcast layers.
7850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7851 EXPECT_THAT(video_source_.sink_wants().resolutions,
7852 ::testing::ElementsAreArray({kLayer0Size}));
7853
7854 video_stream_encoder_->Stop();
7855}
7856
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007857TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007858 ResetEncoder("VP8", 1, 1, 1, false);
7859
Niels Möller8b692902021-06-14 12:04:57 +02007860 // Force encoder reconfig.
7861 video_source_.IncomingCapturedFrame(
7862 CreateFrame(1, codec_width_, codec_height_));
7863 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7864
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007865 // Set QP on encoded frame and pass the frame to encode complete callback.
7866 // Since QP is present QP parsing won't be triggered and the original value
7867 // should be kept.
7868 EncodedImage encoded_image;
7869 encoded_image.qp_ = 123;
7870 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7871 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7872 CodecSpecificInfo codec_info;
7873 codec_info.codecType = kVideoCodecVP8;
7874 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7875 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7876 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7877 video_stream_encoder_->Stop();
7878}
7879
7880TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007881 ResetEncoder("VP8", 1, 1, 1, false);
7882
Niels Möller8b692902021-06-14 12:04:57 +02007883 // Force encoder reconfig.
7884 video_source_.IncomingCapturedFrame(
7885 CreateFrame(1, codec_width_, codec_height_));
7886 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7887
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007888 // Pass an encoded frame without QP to encode complete callback. QP should be
7889 // parsed and set.
7890 EncodedImage encoded_image;
7891 encoded_image.qp_ = -1;
7892 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7893 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7894 CodecSpecificInfo codec_info;
7895 codec_info.codecType = kVideoCodecVP8;
7896 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7897 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7898 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7899 video_stream_encoder_->Stop();
7900}
7901
7902TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7903 webrtc::test::ScopedFieldTrials field_trials(
7904 "WebRTC-QpParsingKillSwitch/Enabled/");
7905
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007906 ResetEncoder("VP8", 1, 1, 1, false);
7907
Niels Möller8b692902021-06-14 12:04:57 +02007908 // Force encoder reconfig.
7909 video_source_.IncomingCapturedFrame(
7910 CreateFrame(1, codec_width_, codec_height_));
7911 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7912
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007913 EncodedImage encoded_image;
7914 encoded_image.qp_ = -1;
7915 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7916 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7917 CodecSpecificInfo codec_info;
7918 codec_info.codecType = kVideoCodecVP8;
7919 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7920 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7921 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7922 video_stream_encoder_->Stop();
7923}
7924
Sergey Silkind19e3b92021-03-16 10:05:30 +00007925TEST_F(VideoStreamEncoderTest,
7926 QualityScalingNotAllowed_QualityScalingDisabled) {
7927 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7928
7929 // Disable scaling settings in encoder info.
7930 fake_encoder_.SetQualityScaling(false);
7931 // Disable quality scaling in encoder config.
7932 video_encoder_config.is_quality_scaling_allowed = false;
7933 ConfigureEncoder(std::move(video_encoder_config));
7934
7935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007936 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007937
7938 test::FrameForwarder source;
7939 video_stream_encoder_->SetSource(
7940 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7941 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7942 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7943
7944 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7945 WaitForEncodedFrame(1);
7946 video_stream_encoder_->TriggerQualityLow();
7947 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7948
7949 video_stream_encoder_->Stop();
7950}
7951
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007952TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
7953 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7954
7955 // Disable scaling settings in encoder info.
7956 fake_encoder_.SetQualityScaling(false);
7957 // Set QP trusted in encoder info.
7958 fake_encoder_.SetIsQpTrusted(true);
7959 // Enable quality scaling in encoder config.
7960 video_encoder_config.is_quality_scaling_allowed = false;
7961 ConfigureEncoder(std::move(video_encoder_config));
7962
7963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007964 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007965
7966 test::FrameForwarder source;
7967 video_stream_encoder_->SetSource(
7968 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7969 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7970 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7971
7972 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7973 WaitForEncodedFrame(1);
7974 video_stream_encoder_->TriggerQualityLow();
7975 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7976
7977 video_stream_encoder_->Stop();
7978}
7979
Shuhai Pengf2707702021-09-29 17:19:44 +08007980TEST_F(VideoStreamEncoderTest,
7981 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
7982 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7983
7984 // Disable scaling settings in encoder info.
7985 fake_encoder_.SetQualityScaling(false);
7986 // Set QP trusted in encoder info.
7987 fake_encoder_.SetIsQpTrusted(true);
7988 // Enable quality scaling in encoder config.
7989 video_encoder_config.is_quality_scaling_allowed = false;
7990 ConfigureEncoder(std::move(video_encoder_config));
7991
7992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007993 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08007994
7995 test::FrameForwarder source;
7996 video_stream_encoder_->SetSource(
7997 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7998 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7999 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8000
8001 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8002 WaitForEncodedFrame(1);
8003 video_stream_encoder_->TriggerQualityLow();
8004 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8005
8006 video_stream_encoder_->Stop();
8007}
8008
8009TEST_F(VideoStreamEncoderTest,
8010 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8011 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8012
8013 // Disable scaling settings in encoder info.
8014 fake_encoder_.SetQualityScaling(false);
8015 // Set QP trusted in encoder info.
8016 fake_encoder_.SetIsQpTrusted(false);
8017 // Enable quality scaling in encoder config.
8018 video_encoder_config.is_quality_scaling_allowed = false;
8019 ConfigureEncoder(std::move(video_encoder_config));
8020
8021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008023
8024 test::FrameForwarder source;
8025 video_stream_encoder_->SetSource(
8026 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8027 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8028 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8029
8030 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8031 WaitForEncodedFrame(1);
8032 video_stream_encoder_->TriggerQualityLow();
8033 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8034
8035 video_stream_encoder_->Stop();
8036}
8037
8038TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8039 // Set QP trusted in encoder info.
8040 fake_encoder_.SetIsQpTrusted(false);
8041
8042 const int MinEncBitrateKbps = 30;
8043 const int MaxEncBitrateKbps = 100;
8044 const int MinStartBitrateKbp = 50;
8045 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8046 /*frame_size_pixels=*/codec_width_ * codec_height_,
8047 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8048 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8049 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8050
8051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008053
8054 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8055
8056 VideoEncoderConfig video_encoder_config;
8057 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8058 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8059 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8060 MinEncBitrateKbps * 1000;
8061 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8062 kMaxPayloadLength);
8063
8064 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8065 WaitForEncodedFrame(1);
8066 EXPECT_EQ(
8067 MaxEncBitrateKbps,
8068 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8069 EXPECT_EQ(
8070 MinEncBitrateKbps,
8071 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8072
8073 video_stream_encoder_->Stop();
8074}
8075
8076TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8077 // Set QP trusted in encoder info.
8078 fake_encoder_.SetIsQpTrusted(false);
8079
8080 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8081 EncoderInfoSettings::
8082 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8083 codec_width_ * codec_height_,
8084 EncoderInfoSettings::
8085 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8086 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8087
8088 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8089 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8090 const int TargetEncBitrate = MaxEncBitrate;
8091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8092 DataRate::BitsPerSec(TargetEncBitrate),
8093 DataRate::BitsPerSec(TargetEncBitrate),
8094 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8095
8096 VideoEncoderConfig video_encoder_config;
8097 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8098 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8099 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8100 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8101 kMaxPayloadLength);
8102
8103 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8104 WaitForEncodedFrame(1);
8105 EXPECT_EQ(
8106 MaxEncBitrate / 1000,
8107 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8108 EXPECT_EQ(
8109 MinEncBitrate / 1000,
8110 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8111
8112 video_stream_encoder_->Stop();
8113}
8114
Sergey Silkind19e3b92021-03-16 10:05:30 +00008115#if !defined(WEBRTC_IOS)
8116// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8117// disabled by default on iOS.
8118TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8119 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8120
8121 // Disable scaling settings in encoder info.
8122 fake_encoder_.SetQualityScaling(false);
8123 // Enable quality scaling in encoder config.
8124 video_encoder_config.is_quality_scaling_allowed = true;
8125 ConfigureEncoder(std::move(video_encoder_config));
8126
8127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008128 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008129
8130 test::FrameForwarder source;
8131 video_stream_encoder_->SetSource(
8132 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8133 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8134 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8135
8136 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8137 WaitForEncodedFrame(1);
8138 video_stream_encoder_->TriggerQualityLow();
8139 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8140
8141 video_stream_encoder_->Stop();
8142}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008143
8144TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8145 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8146
8147 // Disable scaling settings in encoder info.
8148 fake_encoder_.SetQualityScaling(false);
8149 // Set QP trusted in encoder info.
8150 fake_encoder_.SetIsQpTrusted(true);
8151 // Enable quality scaling in encoder config.
8152 video_encoder_config.is_quality_scaling_allowed = true;
8153 ConfigureEncoder(std::move(video_encoder_config));
8154
8155 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008156 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008157
8158 test::FrameForwarder source;
8159 video_stream_encoder_->SetSource(
8160 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8161 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8162 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8163
8164 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8165 WaitForEncodedFrame(1);
8166 video_stream_encoder_->TriggerQualityLow();
8167 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8168
8169 video_stream_encoder_->Stop();
8170}
Shuhai Pengf2707702021-09-29 17:19:44 +08008171
8172TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8173 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8174
8175 // Disable scaling settings in encoder info.
8176 fake_encoder_.SetQualityScaling(false);
8177 // Set QP not trusted in encoder info.
8178 fake_encoder_.SetIsQpTrusted(false);
8179 // Enable quality scaling in encoder config.
8180 video_encoder_config.is_quality_scaling_allowed = true;
8181 ConfigureEncoder(std::move(video_encoder_config));
8182
8183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008184 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008185
8186 test::FrameForwarder source;
8187 video_stream_encoder_->SetSource(
8188 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8189 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8190 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8191
8192 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8193 WaitForEncodedFrame(1);
8194 video_stream_encoder_->TriggerQualityLow();
8195 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8196 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8197 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8198
8199 video_stream_encoder_->Stop();
8200}
8201
8202TEST_F(VideoStreamEncoderTest,
8203 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8204 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8205
8206 // Disable scaling settings in encoder info.
8207 fake_encoder_.SetQualityScaling(false);
8208 // Set QP trusted in encoder info.
8209 fake_encoder_.SetIsQpTrusted(true);
8210 // Enable quality scaling in encoder config.
8211 video_encoder_config.is_quality_scaling_allowed = true;
8212 ConfigureEncoder(std::move(video_encoder_config));
8213
8214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008215 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008216
8217 test::FrameForwarder source;
8218 video_stream_encoder_->SetSource(
8219 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8220 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8221 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8222
8223 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8224 WaitForEncodedFrame(1);
8225 video_stream_encoder_->TriggerQualityLow();
8226 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8227 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8228
8229 video_stream_encoder_->Stop();
8230}
8231
8232TEST_F(VideoStreamEncoderTest,
8233 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8234 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8235
8236 // Disable scaling settings in encoder info.
8237 fake_encoder_.SetQualityScaling(false);
8238 // Set QP trusted in encoder info.
8239 fake_encoder_.SetIsQpTrusted(false);
8240 // Enable quality scaling in encoder config.
8241 video_encoder_config.is_quality_scaling_allowed = true;
8242 ConfigureEncoder(std::move(video_encoder_config));
8243
8244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008245 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008246
8247 test::FrameForwarder source;
8248 video_stream_encoder_->SetSource(
8249 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8250 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8252
8253 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8254 WaitForEncodedFrame(1);
8255 video_stream_encoder_->TriggerQualityLow();
8256 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8257
8258 video_stream_encoder_->Stop();
8259}
8260
Sergey Silkind19e3b92021-03-16 10:05:30 +00008261#endif
8262
Henrik Boström56db9ff2021-03-24 09:06:45 +01008263// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8264class VideoStreamEncoderWithRealEncoderTest
8265 : public VideoStreamEncoderTest,
8266 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8267 public:
8268 VideoStreamEncoderWithRealEncoderTest()
8269 : VideoStreamEncoderTest(),
8270 codec_type_(std::get<0>(GetParam())),
8271 allow_i420_conversion_(std::get<1>(GetParam())) {}
8272
8273 void SetUp() override {
8274 VideoStreamEncoderTest::SetUp();
8275 std::unique_ptr<VideoEncoder> encoder;
8276 switch (codec_type_) {
8277 case kVideoCodecVP8:
8278 encoder = VP8Encoder::Create();
8279 break;
8280 case kVideoCodecVP9:
8281 encoder = VP9Encoder::Create();
8282 break;
8283 case kVideoCodecAV1:
8284 encoder = CreateLibaomAv1Encoder();
8285 break;
8286 case kVideoCodecH264:
8287 encoder =
8288 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8289 break;
8290 case kVideoCodecMultiplex:
8291 mock_encoder_factory_for_multiplex_ =
8292 std::make_unique<MockVideoEncoderFactory>();
8293 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8294 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8295 .WillRepeatedly([] { return VP8Encoder::Create(); });
8296 encoder = std::make_unique<MultiplexEncoderAdapter>(
8297 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8298 false);
8299 break;
8300 default:
Artem Titovd3251962021-11-15 16:57:07 +01008301 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008302 }
8303 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8304 }
8305
8306 void TearDown() override {
8307 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008308 // Ensure `video_stream_encoder_` is destroyed before
8309 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008310 video_stream_encoder_.reset();
8311 VideoStreamEncoderTest::TearDown();
8312 }
8313
8314 protected:
8315 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8316 std::unique_ptr<VideoEncoder> encoder) {
8317 // Configure VSE to use the encoder.
8318 encoder_ = std::move(encoder);
8319 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8320 encoder_.get(), &encoder_selector_);
8321 video_send_config_.encoder_settings.encoder_factory =
8322 encoder_proxy_factory_.get();
8323 VideoEncoderConfig video_encoder_config;
8324 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8325 video_encoder_config_ = video_encoder_config.Copy();
8326 ConfigureEncoder(video_encoder_config_.Copy());
8327
8328 // Set bitrate to ensure frame is not dropped.
8329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008330 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008331 }
8332
8333 const VideoCodecType codec_type_;
8334 const bool allow_i420_conversion_;
8335 NiceMock<MockEncoderSelector> encoder_selector_;
8336 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8337 std::unique_ptr<VideoEncoder> encoder_;
8338 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8339};
8340
8341TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8342 auto native_i420_frame = test::CreateMappableNativeFrame(
8343 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8344 video_source_.IncomingCapturedFrame(native_i420_frame);
8345 WaitForEncodedFrame(codec_width_, codec_height_);
8346
8347 auto mappable_native_buffer =
8348 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8349 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8350 mappable_native_buffer->GetMappedFramedBuffers();
8351 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8352 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8353 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8354 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8355}
8356
8357TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8358 auto native_nv12_frame = test::CreateMappableNativeFrame(
8359 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8360 video_source_.IncomingCapturedFrame(native_nv12_frame);
8361 WaitForEncodedFrame(codec_width_, codec_height_);
8362
8363 auto mappable_native_buffer =
8364 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8365 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8366 mappable_native_buffer->GetMappedFramedBuffers();
8367 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8368 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8369 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8370 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8371
8372 if (!allow_i420_conversion_) {
8373 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8374 }
8375}
8376
Erik Språng7444b192021-06-02 14:02:13 +02008377TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8378 if (codec_type_ == kVideoCodecMultiplex) {
8379 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8380 return;
8381 }
8382
8383 const size_t kNumSpatialLayers = 3u;
8384 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8385 const int kFrameWidth = 1280;
8386 const int kFrameHeight = 720;
8387 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8388 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8389 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8390 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8391 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8392 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8393
8394 VideoEncoderConfig config;
8395 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8396 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008397 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008398 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8399 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8400 vp9_settings.numberOfTemporalLayers = 3;
8401 vp9_settings.automaticResizeOn = false;
8402 config.encoder_specific_settings =
8403 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8404 vp9_settings);
8405 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8406 /*fps=*/30.0,
8407 /*first_active_layer=*/0,
8408 /*num_spatial_layers=*/3,
8409 /*num_temporal_layers=*/3,
8410 /*is_screenshare=*/false);
8411 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8412 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008413 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008414 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8415 /*fps=*/30.0,
8416 /*first_active_layer=*/0,
8417 /*num_spatial_layers=*/3,
8418 /*num_temporal_layers=*/3,
8419 /*is_screenshare=*/false);
8420 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8421 } else {
8422 // Simulcast for VP8/H264.
8423 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8424 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8425 config.simulcast_layers[i].scale_resolution_down_by =
8426 kDownscaleFactors[i];
8427 config.simulcast_layers[i].active = true;
8428 }
8429 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8430 // Turn off frame dropping to prevent flakiness.
8431 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8432 h264_settings.frameDroppingOn = false;
8433 config.encoder_specific_settings = rtc::make_ref_counted<
8434 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8435 }
8436 }
8437
8438 auto set_layer_active = [&](int layer_idx, bool active) {
8439 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8440 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8441 config.spatial_layers[layer_idx].active = active;
8442 } else {
8443 config.simulcast_layers[layer_idx].active = active;
8444 }
8445 };
8446
8447 config.video_stream_factory =
8448 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8449 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8450 /*screencast*/ false,
8451 /*screenshare enabled*/ false);
8452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008453 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8454 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008455
8456 // Capture a frame with all layers active.
8457 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8458 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8459 int64_t timestamp_ms = kFrameIntervalMs;
8460 video_source_.IncomingCapturedFrame(
8461 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8462
8463 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8465
8466 // Capture a frame with one of the layers inactive.
8467 set_layer_active(2, false);
8468 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8469 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8470 timestamp_ms += kFrameIntervalMs;
8471 video_source_.IncomingCapturedFrame(
8472 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8473 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8474
8475 // New target bitrates signaled based on lower resolution.
8476 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8478 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8479 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8480
8481 // Re-enable the top layer.
8482 set_layer_active(2, true);
8483 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8484 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8485 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8486
8487 // Bitrate target adjusted back up to enable HD layer...
8488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8489 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8490 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8491 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8492
8493 // ...then add a new frame.
8494 timestamp_ms += kFrameIntervalMs;
8495 video_source_.IncomingCapturedFrame(
8496 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8497 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8498 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8499
8500 video_stream_encoder_->Stop();
8501}
8502
Henrik Boström56db9ff2021-03-24 09:06:45 +01008503std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8504 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8505 VideoCodecType codec_type = std::get<0>(info.param);
8506 bool allow_i420_conversion = std::get<1>(info.param);
8507 std::string str;
8508 switch (codec_type) {
8509 case kVideoCodecGeneric:
8510 str = "Generic";
8511 break;
8512 case kVideoCodecVP8:
8513 str = "VP8";
8514 break;
8515 case kVideoCodecVP9:
8516 str = "VP9";
8517 break;
8518 case kVideoCodecAV1:
8519 str = "AV1";
8520 break;
8521 case kVideoCodecH264:
8522 str = "H264";
8523 break;
8524 case kVideoCodecMultiplex:
8525 str = "Multiplex";
8526 break;
8527 default:
Artem Titovd3251962021-11-15 16:57:07 +01008528 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008529 }
8530 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8531 return str;
8532}
8533
8534constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8535 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8536constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8537 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8538constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8539 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8540constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8541 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8542#if defined(WEBRTC_USE_H264)
8543constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8544 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8545
8546// The windows compiler does not tolerate #if statements inside the
8547// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8548// and without H264).
8549INSTANTIATE_TEST_SUITE_P(
8550 All,
8551 VideoStreamEncoderWithRealEncoderTest,
8552 ::testing::Values(kVP8DisallowConversion,
8553 kVP9DisallowConversion,
8554 kAV1AllowConversion,
8555 kMultiplexDisallowConversion,
8556 kH264AllowConversion),
8557 TestParametersVideoCodecAndAllowI420ConversionToString);
8558#else
8559INSTANTIATE_TEST_SUITE_P(
8560 All,
8561 VideoStreamEncoderWithRealEncoderTest,
8562 ::testing::Values(kVP8DisallowConversion,
8563 kVP9DisallowConversion,
8564 kAV1AllowConversion,
8565 kMultiplexDisallowConversion),
8566 TestParametersVideoCodecAndAllowI420ConversionToString);
8567#endif
8568
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008569class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8570 protected:
8571 void RunTest(const std::vector<VideoStream>& configs,
8572 const int expected_num_init_encode) {
8573 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008574 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008575 InsertFrameAndWaitForEncoded();
8576 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8577 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008578 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8579 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008580
8581 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8582 ConfigureEncoder(configs[1]);
8583 InsertFrameAndWaitForEncoded();
8584 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8585 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008586 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008587 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008588 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008589
8590 video_stream_encoder_->Stop();
8591 }
8592
8593 void ConfigureEncoder(const VideoStream& stream) {
8594 VideoEncoderConfig config;
8595 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8596 config.max_bitrate_bps = stream.max_bitrate_bps;
8597 config.simulcast_layers[0] = stream;
8598 config.video_stream_factory =
8599 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8600 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8601 /*conference_mode=*/false);
8602 video_stream_encoder_->ConfigureEncoder(std::move(config),
8603 kMaxPayloadLength);
8604 }
8605
8606 void OnBitrateUpdated(DataRate bitrate) {
8607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8608 bitrate, bitrate, bitrate, 0, 0, 0);
8609 }
8610
8611 void InsertFrameAndWaitForEncoded() {
8612 timestamp_ms_ += kFrameIntervalMs;
8613 video_source_.IncomingCapturedFrame(
8614 CreateFrame(timestamp_ms_, kWidth, kHeight));
8615 sink_.WaitForEncodedFrame(timestamp_ms_);
8616 }
8617
8618 void ExpectEqual(const VideoCodec& actual,
8619 const VideoStream& expected) const {
8620 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8621 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8622 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8623 static_cast<unsigned int>(expected.min_bitrate_bps));
8624 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8625 static_cast<unsigned int>(expected.max_bitrate_bps));
8626 EXPECT_EQ(actual.simulcastStream[0].width,
8627 kWidth / expected.scale_resolution_down_by);
8628 EXPECT_EQ(actual.simulcastStream[0].height,
8629 kHeight / expected.scale_resolution_down_by);
8630 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8631 expected.num_temporal_layers);
8632 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8633 }
8634
8635 VideoStream DefaultConfig() const {
8636 VideoStream stream;
8637 stream.max_framerate = 25;
8638 stream.min_bitrate_bps = 35000;
8639 stream.max_bitrate_bps = 900000;
8640 stream.scale_resolution_down_by = 1.0;
8641 stream.num_temporal_layers = 1;
8642 stream.bitrate_priority = 1.0;
8643 stream.scalability_mode = "";
8644 return stream;
8645 }
8646
8647 const int kWidth = 640;
8648 const int kHeight = 360;
8649 int64_t timestamp_ms_ = 0;
8650};
8651
8652TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8653 VideoStream config1 = DefaultConfig();
8654 VideoStream config2 = config1;
8655 config2.max_framerate++;
8656
8657 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8658}
8659
8660TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8661 VideoStream config1 = DefaultConfig();
8662 VideoStream config2 = config1;
8663 config2.min_bitrate_bps += 10000;
8664
8665 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8666}
8667
8668TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8669 VideoStream config1 = DefaultConfig();
8670 VideoStream config2 = config1;
8671 config2.max_bitrate_bps += 100000;
8672
8673 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8674}
8675
8676TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8677 VideoStream config1 = DefaultConfig();
8678 VideoStream config2 = config1;
8679 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8680
8681 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8682}
8683
8684TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8685 VideoStream config1 = DefaultConfig();
8686 VideoStream config2 = config1;
8687 config2.scale_resolution_down_by *= 2;
8688
8689 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8690}
8691
8692TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8693 VideoStream config1 = DefaultConfig();
8694 VideoStream config2 = config1;
8695 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8696
8697 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8698}
8699
8700TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8701 VideoStream config1 = DefaultConfig();
8702 VideoStream config2 = config1;
8703 config2.scalability_mode = "L1T2";
8704
8705 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8706}
8707
Markus Handellb4e96d42021-11-05 12:00:55 +01008708TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8709 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8710 auto* adapter_ptr = adapter.get();
8711 SimpleVideoStreamEncoderFactory factory;
8712 auto video_stream_encoder = factory.Create(std::move(adapter));
8713
8714 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true));
8715 VideoEncoderConfig config;
8716 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8717 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handell9a478b52021-11-18 16:07:01 +01008718 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008719 Mock::VerifyAndClearExpectations(adapter_ptr);
8720
8721 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false));
8722 VideoEncoderConfig config2;
8723 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8724 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell9a478b52021-11-18 16:07:01 +01008725 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008726}
8727
8728TEST(VideoStreamEncoderFrameCadenceTest,
8729 ForwardsFramesIntoFrameCadenceAdapter) {
8730 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8731 auto* adapter_ptr = adapter.get();
8732 test::FrameForwarder video_source;
8733 SimpleVideoStreamEncoderFactory factory;
8734 auto video_stream_encoder = factory.Create(std::move(adapter));
8735 video_stream_encoder->SetSource(
8736 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8737
8738 EXPECT_CALL(*adapter_ptr, OnFrame);
8739 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8740 video_source.IncomingCapturedFrame(
8741 VideoFrame::Builder()
8742 .set_video_frame_buffer(std::move(buffer))
8743 .set_ntp_time_ms(0)
8744 .set_timestamp_ms(0)
8745 .set_rotation(kVideoRotation_0)
8746 .build());
8747}
8748
perkj26091b12016-09-01 01:17:40 -07008749} // namespace webrtc