blob: 9782e8358e22ff8fce68a76320b7e1d0caf4854c [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020045#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020046#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010047#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020048#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010049#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020050#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020053#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010054#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010062#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020063#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020064#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070066
67namespace webrtc {
68
sprang57c2fff2017-01-16 06:24:02 -080069using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020071using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020072using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010081using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080083
perkj803d97f2016-11-01 11:45:46 -070084namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080091const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020092const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
93const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
94const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
95const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -080096const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070097const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020099const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800104
Asa Persson606d3cb2021-10-04 10:07:11 +0200105uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 11:45:46 -0700115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 09:06:45 +0100129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150 }
Noah Richards51db4212019-06-12 06:59:12 -0700151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 09:15:13 +0200163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 10:27:48 +0100194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 09:04:13 +0200215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 18:49:07 +0200217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219 private:
Markus Handella3765182020-07-08 13:13:32 +0200220 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100223};
224
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200227 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 08:27:51 -0700343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700344 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100358 task_queue_factory,
359 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200360 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200361 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200362 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200363 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200364 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200365 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200366 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200367 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100368 }
perkj803d97f2016-11-01 11:45:46 -0700369
Henrik Boström381d1092020-05-12 18:49:07 +0200370 void SetSourceAndWaitForRestrictionsUpdated(
371 rtc::VideoSourceInterface<VideoFrame>* source,
372 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 FakeVideoSourceRestrictionsListener listener;
374 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200375 SetSource(source, degradation_preference);
376 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200377 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200378 }
379
380 void SetSourceAndWaitForFramerateUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
383 overuse_detector_proxy_->framerate_updated_event()->Reset();
384 SetSource(source, degradation_preference);
385 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
386 }
387
388 void OnBitrateUpdatedAndWaitForManagedResources(
389 DataRate target_bitrate,
390 DataRate stable_target_bitrate,
391 DataRate link_allocation,
392 uint8_t fraction_lost,
393 int64_t round_trip_time_ms,
394 double cwnd_reduce_ratio) {
395 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
396 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
397 // Bitrate is updated on the encoder queue.
398 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200399 }
400
kthelgason2fc52542017-03-03 00:24:41 -0800401 // This is used as a synchronisation mechanism, to make sure that the
402 // encoder queue is not blocked before we start sending it frames.
403 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100404 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200405 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800406 ASSERT_TRUE(event.Wait(5000));
407 }
408
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200419
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 void TriggerCpuUnderuse() {
421 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200422 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200423 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 event.Set();
425 });
426 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200427 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200428 }
kthelgason876222f2016-11-29 01:44:11 -0800429
Henrik Boström91aa7322020-04-28 12:24:33 +0200430 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200431 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200433 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200434 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200435 event.Set();
436 });
437 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200438 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200440 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200443 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 }
449
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100451 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200452 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
453 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700455};
456
Noah Richards51db4212019-06-12 06:59:12 -0700457// Simulates simulcast behavior and makes highest stream resolutions divisible
458// by 4.
459class CroppingVideoStreamFactory
460 : public VideoEncoderConfig::VideoStreamFactoryInterface {
461 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200462 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700463
464 private:
465 std::vector<VideoStream> CreateEncoderStreams(
466 int width,
467 int height,
468 const VideoEncoderConfig& encoder_config) override {
469 std::vector<VideoStream> streams = test::CreateVideoStreams(
470 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700471 return streams;
472 }
Noah Richards51db4212019-06-12 06:59:12 -0700473};
474
sprangb1ca0732017-02-01 08:38:12 -0800475class AdaptingFrameForwarder : public test::FrameForwarder {
476 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 explicit AdaptingFrameForwarder(TimeController* time_controller)
478 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700479 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800480
481 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200482 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800483 adaptation_enabled_ = enabled;
484 }
485
asaperssonfab67072017-04-04 05:51:49 -0700486 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800488 return adaptation_enabled_;
489 }
490
Henrik Boström1124ed12021-02-25 10:30:39 +0100491 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
492 // the resolution or frame rate was different than it is currently. If
493 // something else is modified, such as encoder resolutions, but the resolution
494 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700495 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200496 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700497 return last_wants_;
498 }
499
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200500 absl::optional<int> last_sent_width() const { return last_width_; }
501 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800502
sprangb1ca0732017-02-01 08:38:12 -0800503 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200504 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
505 time_controller_->AdvanceTime(TimeDelta::Millis(0));
506
sprangb1ca0732017-02-01 08:38:12 -0800507 int cropped_width = 0;
508 int cropped_height = 0;
509 int out_width = 0;
510 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700511 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
513 << "w=" << video_frame.width()
514 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700515 if (adapter_.AdaptFrameResolution(
516 video_frame.width(), video_frame.height(),
517 video_frame.timestamp_us() * 1000, &cropped_width,
518 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame adapted_frame =
520 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200521 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100522 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200523 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-02 23:53:04 -0700534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700540 }
sprangb1ca0732017-02-01 08:38:12 -0800541 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800546 }
547 }
548
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200549 void OnOutputFormatRequest(int width, int height) {
550 absl::optional<std::pair<int, int>> target_aspect_ratio =
551 std::make_pair(width, height);
552 absl::optional<int> max_pixel_count = width * height;
553 absl::optional<int> max_fps;
554 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
555 max_fps);
556 }
557
sprangb1ca0732017-02-01 08:38:12 -0800558 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
559 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200560 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100561 rtc::VideoSinkWants prev_wants = sink_wants_locked();
562 bool did_adapt =
563 prev_wants.max_pixel_count != wants.max_pixel_count ||
564 prev_wants.target_pixel_count != wants.target_pixel_count ||
565 prev_wants.max_framerate_fps != wants.max_framerate_fps;
566 if (did_adapt) {
567 last_wants_ = prev_wants;
568 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100569 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200570 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800571 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200572
573 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800574 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200575 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
576 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200577 absl::optional<int> last_width_;
578 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800579};
sprangc5d62e22017-04-02 23:53:04 -0700580
Niels Möller213618e2018-07-24 09:29:58 +0200581// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700582class MockableSendStatisticsProxy : public SendStatisticsProxy {
583 public:
584 MockableSendStatisticsProxy(Clock* clock,
585 const VideoSendStream::Config& config,
586 VideoEncoderConfig::ContentType content_type)
587 : SendStatisticsProxy(clock, config, content_type) {}
588
589 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200590 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700591 if (mock_stats_)
592 return *mock_stats_;
593 return SendStatisticsProxy::GetStats();
594 }
595
Niels Möller213618e2018-07-24 09:29:58 +0200596 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200597 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200598 if (mock_stats_)
599 return mock_stats_->input_frame_rate;
600 return SendStatisticsProxy::GetInputFrameRate();
601 }
sprangc5d62e22017-04-02 23:53:04 -0700602 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200603 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700604 mock_stats_.emplace(stats);
605 }
606
607 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200608 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700609 mock_stats_.reset();
610 }
611
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200612 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
613 on_frame_dropped_ = std::move(callback);
614 }
615
sprangc5d62e22017-04-02 23:53:04 -0700616 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 void OnFrameDropped(DropReason reason) override {
618 SendStatisticsProxy::OnFrameDropped(reason);
619 if (on_frame_dropped_)
620 on_frame_dropped_(reason);
621 }
622
Markus Handella3765182020-07-08 13:13:32 +0200623 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200624 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200625 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700626};
627
philipel9b058032020-02-10 11:30:00 +0100628class MockEncoderSelector
629 : public VideoEncoderFactory::EncoderSelectorInterface {
630 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200631 MOCK_METHOD(void,
632 OnCurrentEncoder,
633 (const SdpVideoFormat& format),
634 (override));
635 MOCK_METHOD(absl::optional<SdpVideoFormat>,
636 OnAvailableBitrate,
637 (const DataRate& rate),
638 (override));
639 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100640};
641
perkj803d97f2016-11-01 11:45:46 -0700642} // namespace
643
mflodmancc3d4422017-08-03 08:27:51 -0700644class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700645 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200646 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700647
mflodmancc3d4422017-08-03 08:27:51 -0700648 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700649 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700650 codec_width_(320),
651 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200652 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200654 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700655 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200656 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700657 video_send_config_,
658 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200659 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700660
661 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700662 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700663 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200664 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800665 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200666 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200667 video_send_config_.rtp.payload_name = "FAKE";
668 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700669
Per512ecb32016-09-23 15:52:06 +0200670 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200671 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200672 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
673 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
674 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100675 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700676
Niels Möllerf1338562018-04-26 09:51:47 +0200677 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800678 }
679
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100680 void ConfigureEncoder(
681 VideoEncoderConfig video_encoder_config,
682 VideoStreamEncoder::BitrateAllocationCallbackType
683 allocation_callback_type =
684 VideoStreamEncoder::BitrateAllocationCallbackType::
685 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700686 if (video_stream_encoder_)
687 video_stream_encoder_->Stop();
688 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200689 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100690 video_send_config_.encoder_settings, allocation_callback_type));
Asa Persson606d3cb2021-10-04 10:07:11 +0200691 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700693 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200694 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200696 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800698 }
699
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100700 void ResetEncoder(const std::string& payload_name,
701 size_t num_streams,
702 size_t num_temporal_layers,
703 unsigned char num_spatial_layers,
704 bool screenshare,
705 VideoStreamEncoder::BitrateAllocationCallbackType
706 allocation_callback_type =
707 VideoStreamEncoder::BitrateAllocationCallbackType::
708 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200709 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800710
711 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200712 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
713 num_streams, &video_encoder_config);
714 for (auto& layer : video_encoder_config.simulcast_layers) {
715 layer.num_temporal_layers = num_temporal_layers;
716 layer.max_framerate = kDefaultFramerate;
717 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100718 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200719 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700720 video_encoder_config.content_type =
721 screenshare ? VideoEncoderConfig::ContentType::kScreen
722 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700723 if (payload_name == "VP9") {
724 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
725 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200726 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200728 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
729 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700730 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100731 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700732 }
733
sprang57c2fff2017-01-16 06:24:02 -0800734 VideoFrame CreateFrame(int64_t ntp_time_ms,
735 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200736 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200737 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200738 destruction_event, codec_width_, codec_height_))
739 .set_ntp_time_ms(ntp_time_ms)
740 .set_timestamp_ms(99)
741 .set_rotation(kVideoRotation_0)
742 .build();
perkj26091b12016-09-01 01:17:40 -0700743 }
744
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100745 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200748 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200749 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200750 destruction_event, codec_width_, codec_height_))
751 .set_ntp_time_ms(ntp_time_ms)
752 .set_timestamp_ms(99)
753 .set_rotation(kVideoRotation_0)
754 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
755 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100756 }
757
sprang57c2fff2017-01-16 06:24:02 -0800758 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200759 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
760 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200761 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200762 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200763 .set_ntp_time_ms(ntp_time_ms)
764 .set_timestamp_ms(ntp_time_ms)
765 .set_rotation(kVideoRotation_0)
766 .build();
perkj803d97f2016-11-01 11:45:46 -0700767 }
768
Evan Shrubsole895556e2020-10-05 09:15:13 +0200769 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200770 return VideoFrame::Builder()
771 .set_video_frame_buffer(NV12Buffer::Create(width, height))
772 .set_ntp_time_ms(ntp_time_ms)
773 .set_timestamp_ms(ntp_time_ms)
774 .set_rotation(kVideoRotation_0)
775 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200776 }
777
Noah Richards51db4212019-06-12 06:59:12 -0700778 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200782 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200783 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200784 destruction_event, width, height))
785 .set_ntp_time_ms(ntp_time_ms)
786 .set_timestamp_ms(99)
787 .set_rotation(kVideoRotation_0)
788 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700789 }
790
Evan Shrubsole895556e2020-10-05 09:15:13 +0200791 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
792 rtc::Event* destruction_event,
793 int width,
794 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200795 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200796 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200797 destruction_event, width, height))
798 .set_ntp_time_ms(ntp_time_ms)
799 .set_timestamp_ms(99)
800 .set_rotation(kVideoRotation_0)
801 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200802 }
803
Noah Richards51db4212019-06-12 06:59:12 -0700804 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
805 rtc::Event* destruction_event) const {
806 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
807 codec_height_);
808 }
809
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100810 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200812 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100813
814 video_source_.IncomingCapturedFrame(
815 CreateFrame(1, codec_width_, codec_height_));
816 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200817 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100818 }
819
sprang4847ae62017-06-27 07:06:52 -0700820 void WaitForEncodedFrame(int64_t expected_ntp_time) {
821 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200822 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700823 }
824
825 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
826 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200827 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700828 return ok;
829 }
830
831 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
832 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200833 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700834 }
835
836 void ExpectDroppedFrame() {
837 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200838 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700839 }
840
841 bool WaitForFrame(int64_t timeout_ms) {
842 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200843 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700844 return ok;
845 }
846
perkj26091b12016-09-01 01:17:40 -0700847 class TestEncoder : public test::FakeEncoder {
848 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200849 explicit TestEncoder(TimeController* time_controller)
850 : FakeEncoder(time_controller->GetClock()),
851 time_controller_(time_controller) {
852 RTC_DCHECK(time_controller_);
853 }
perkj26091b12016-09-01 01:17:40 -0700854
perkjfa10b552016-10-02 23:45:26 -0700855 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200856 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700857 block_next_encode_ = true;
858 }
859
Erik Språngaed30702018-11-05 12:57:17 +0100860 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200861 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200862 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100863 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100864 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100865 info.scaling_settings = VideoEncoder::ScalingSettings(
866 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100867 }
868 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100869 for (int i = 0; i < kMaxSpatialLayers; ++i) {
870 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100871 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100872 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100873 for (int tid = 0; tid < num_layers; ++tid)
874 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100875 }
876 }
Erik Språngaed30702018-11-05 12:57:17 +0100877 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200878
879 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100880 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200881 info.apply_alignment_to_all_simulcast_layers =
882 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200883 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800884 if (is_qp_trusted_.has_value()) {
885 info.is_qp_trusted = is_qp_trusted_;
886 }
Erik Språngaed30702018-11-05 12:57:17 +0100887 return info;
kthelgason876222f2016-11-29 01:44:11 -0800888 }
889
Erik Språngb7cb7b52019-02-26 15:52:33 +0100890 int32_t RegisterEncodeCompleteCallback(
891 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200892 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100893 encoded_image_callback_ = callback;
894 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
895 }
896
perkjfa10b552016-10-02 23:45:26 -0700897 void ContinueEncode() { continue_encode_event_.Set(); }
898
899 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
900 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200901 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700902 EXPECT_EQ(timestamp_, timestamp);
903 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
904 }
905
kthelgason2fc52542017-03-03 00:24:41 -0800906 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200907 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800908 quality_scaling_ = b;
909 }
kthelgasonad9010c2017-02-14 00:46:51 -0800910
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100911 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200912 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100913 requested_resolution_alignment_ = requested_resolution_alignment;
914 }
915
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200916 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
917 MutexLock lock(&local_mutex_);
918 apply_alignment_to_all_simulcast_layers_ = b;
919 }
920
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100921 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200922 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100923 is_hardware_accelerated_ = is_hardware_accelerated;
924 }
925
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100926 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
927 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200928 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100929 temporal_layers_supported_[spatial_idx] = supported;
930 }
931
Sergey Silkin6456e352019-07-08 17:56:40 +0200932 void SetResolutionBitrateLimits(
933 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200934 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200935 resolution_bitrate_limits_ = thresholds;
936 }
937
sprangfe627f32017-03-29 08:24:59 -0700938 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200939 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700940 force_init_encode_failed_ = force_failure;
941 }
942
Niels Möller6bb5ab92019-01-11 11:11:10 +0100943 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200944 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100945 rate_factor_ = rate_factor;
946 }
947
Erik Språngd7329ca2019-02-21 21:19:53 +0100948 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200949 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 return last_framerate_;
951 }
952
Erik Språngd7329ca2019-02-21 21:19:53 +0100953 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200954 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100955 return last_update_rect_;
956 }
957
Niels Möller87e2d782019-03-07 10:18:23 +0100958 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200959 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100960 return last_frame_types_;
961 }
962
963 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100964 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100965 keyframe ? VideoFrameType::kVideoFrameKey
966 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100967 {
Markus Handella3765182020-07-08 13:13:32 +0200968 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100969 last_frame_types_ = frame_type;
970 }
Niels Möllerb859b322019-03-07 12:40:01 +0100971 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100972 }
973
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100974 void InjectEncodedImage(const EncodedImage& image,
975 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200976 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100977 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100978 }
979
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200980 void SetEncodedImageData(
981 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200982 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200983 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200984 }
985
Erik Språngd7329ca2019-02-21 21:19:53 +0100986 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200987 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100988 expect_null_frame_ = true;
989 }
990
Erik Språng5056af02019-09-02 15:53:11 +0200991 absl::optional<VideoEncoder::RateControlParameters>
992 GetAndResetLastRateControlSettings() {
993 auto settings = last_rate_control_settings_;
994 last_rate_control_settings_.reset();
995 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100996 }
997
Henrik Boström56db9ff2021-03-24 09:06:45 +0100998 int GetLastInputWidth() const {
999 MutexLock lock(&local_mutex_);
1000 return last_input_width_;
1001 }
1002
1003 int GetLastInputHeight() const {
1004 MutexLock lock(&local_mutex_);
1005 return last_input_height_;
1006 }
1007
Evan Shrubsole895556e2020-10-05 09:15:13 +02001008 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1009 MutexLock lock(&local_mutex_);
1010 return last_input_pixel_format_;
1011 }
1012
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001013 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001014 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001015 return num_set_rates_;
1016 }
1017
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001018 void SetPreferredPixelFormats(
1019 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1020 pixel_formats) {
1021 MutexLock lock(&local_mutex_);
1022 preferred_pixel_formats_ = std::move(pixel_formats);
1023 }
1024
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001025 void SetIsQpTrusted(absl::optional<bool> trusted) {
1026 MutexLock lock(&local_mutex_);
1027 is_qp_trusted_ = trusted;
1028 }
1029
perkjfa10b552016-10-02 23:45:26 -07001030 private:
perkj26091b12016-09-01 01:17:40 -07001031 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001032 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001033 bool block_encode;
1034 {
Markus Handella3765182020-07-08 13:13:32 +02001035 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001036 if (expect_null_frame_) {
1037 EXPECT_EQ(input_image.timestamp(), 0u);
1038 EXPECT_EQ(input_image.width(), 1);
1039 last_frame_types_ = *frame_types;
1040 expect_null_frame_ = false;
1041 } else {
1042 EXPECT_GT(input_image.timestamp(), timestamp_);
1043 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1044 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1045 }
perkj26091b12016-09-01 01:17:40 -07001046
1047 timestamp_ = input_image.timestamp();
1048 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001049 last_input_width_ = input_image.width();
1050 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001051 block_encode = block_next_encode_;
1052 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001053 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001054 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001055 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001056 }
Niels Möllerb859b322019-03-07 12:40:01 +01001057 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001058 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001059 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001060
perkj26091b12016-09-01 01:17:40 -07001061 return result;
1062 }
1063
Niels Möller08ae7ce2020-09-23 15:58:12 +02001064 CodecSpecificInfo EncodeHook(
1065 EncodedImage& encoded_image,
1066 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001067 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001068 {
1069 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001070 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001071 }
1072 MutexLock lock(&local_mutex_);
1073 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001074 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001075 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001076 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001077 }
1078
sprangfe627f32017-03-29 08:24:59 -07001079 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001080 const Settings& settings) override {
1081 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001082
Markus Handella3765182020-07-08 13:13:32 +02001083 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001084 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001085
Erik Språng82fad3d2018-03-21 09:57:23 +01001086 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001087 // Simulate setting up temporal layers, in order to validate the life
1088 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001089 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001090 frame_buffer_controller_ =
1091 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001092 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001093 if (force_init_encode_failed_) {
1094 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001095 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001096 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001097
Erik Språngb7cb7b52019-02-26 15:52:33 +01001098 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001099 return res;
1100 }
1101
Erik Språngb7cb7b52019-02-26 15:52:33 +01001102 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001103 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001104 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1105 initialized_ = EncoderState::kUninitialized;
1106 return FakeEncoder::Release();
1107 }
1108
Erik Språng16cb8f52019-04-12 13:59:09 +02001109 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001111 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001112 VideoBitrateAllocation adjusted_rate_allocation;
1113 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1114 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001115 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001116 adjusted_rate_allocation.SetBitrate(
1117 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001118 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001119 rate_factor_));
1120 }
1121 }
1122 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001123 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001124 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001125 RateControlParameters adjusted_paramters = parameters;
1126 adjusted_paramters.bitrate = adjusted_rate_allocation;
1127 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001128 }
1129
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001130 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001131 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001132 enum class EncoderState {
1133 kUninitialized,
1134 kInitializationFailed,
1135 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001136 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1137 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001138 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001139 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1140 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1141 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1142 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1143 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1144 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001145 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1146 false;
Markus Handella3765182020-07-08 13:13:32 +02001147 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001148 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1149 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001150 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001151 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001152 absl::optional<bool>
1153 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001154 local_mutex_);
1155 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1156 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1157 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001158 absl::optional<VideoEncoder::RateControlParameters>
1159 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001160 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1161 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001162 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001163 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001164 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1165 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001166 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001167 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001168 RTC_GUARDED_BY(local_mutex_);
1169 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001170 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1171 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001172 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1173 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001174 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001175 };
1176
mflodmancc3d4422017-08-03 08:27:51 -07001177 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001178 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001179 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1180 : time_controller_(time_controller), test_encoder_(test_encoder) {
1181 RTC_DCHECK(time_controller_);
1182 }
perkj26091b12016-09-01 01:17:40 -07001183
perkj26091b12016-09-01 01:17:40 -07001184 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001185 EXPECT_TRUE(
1186 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1187 }
1188
1189 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1190 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001191 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001192 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001193 return false;
perkj26091b12016-09-01 01:17:40 -07001194 {
Markus Handella3765182020-07-08 13:13:32 +02001195 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001196 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001197 }
1198 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001199 return true;
perkj26091b12016-09-01 01:17:40 -07001200 }
1201
sprangb1ca0732017-02-01 08:38:12 -08001202 void WaitForEncodedFrame(uint32_t expected_width,
1203 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001204 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001205 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001206 }
1207
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001208 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001209 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001210 uint32_t width = 0;
1211 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001212 {
Markus Handella3765182020-07-08 13:13:32 +02001213 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001214 width = last_width_;
1215 height = last_height_;
1216 }
1217 EXPECT_EQ(expected_height, height);
1218 EXPECT_EQ(expected_width, width);
1219 }
1220
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001221 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1222 VideoRotation rotation;
1223 {
Markus Handella3765182020-07-08 13:13:32 +02001224 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001225 rotation = last_rotation_;
1226 }
1227 EXPECT_EQ(expected_rotation, rotation);
1228 }
1229
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001230 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001231
sprangc5d62e22017-04-02 23:53:04 -07001232 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001233 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1234 bool ret = encoded_frame_event_.Wait(timeout_ms);
1235 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1236 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001237 }
1238
perkj26091b12016-09-01 01:17:40 -07001239 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001240 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001241 expect_frames_ = false;
1242 }
1243
asaperssonfab67072017-04-04 05:51:49 -07001244 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001245 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001246 return number_of_reconfigurations_;
1247 }
1248
asaperssonfab67072017-04-04 05:51:49 -07001249 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001250 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001251 return min_transmit_bitrate_bps_;
1252 }
1253
Erik Språngd7329ca2019-02-21 21:19:53 +01001254 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001255 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001256 num_expected_layers_ = num_layers;
1257 }
1258
Erik Språngb7cb7b52019-02-26 15:52:33 +01001259 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001260 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001261 return last_capture_time_ms_;
1262 }
1263
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001264 const EncodedImage& GetLastEncodedImage() {
1265 MutexLock lock(&mutex_);
1266 return last_encoded_image_;
1267 }
1268
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001269 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001270 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001271 return std::move(last_encoded_image_data_);
1272 }
1273
Per Kjellanderdcef6412020-10-07 15:09:05 +02001274 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1275 MutexLock lock(&mutex_);
1276 return last_bitrate_allocation_;
1277 }
1278
1279 int number_of_bitrate_allocations() const {
1280 MutexLock lock(&mutex_);
1281 return number_of_bitrate_allocations_;
1282 }
1283
Per Kjellandera9434842020-10-15 17:53:22 +02001284 VideoLayersAllocation GetLastVideoLayersAllocation() {
1285 MutexLock lock(&mutex_);
1286 return last_layers_allocation_;
1287 }
1288
1289 int number_of_layers_allocations() const {
1290 MutexLock lock(&mutex_);
1291 return number_of_layers_allocations_;
1292 }
1293
perkj26091b12016-09-01 01:17:40 -07001294 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001295 Result OnEncodedImage(
1296 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001297 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001298 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001299 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001300 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001301 last_encoded_image_data_ = std::vector<uint8_t>(
1302 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001303 uint32_t timestamp = encoded_image.Timestamp();
1304 if (last_timestamp_ != timestamp) {
1305 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001306 last_width_ = encoded_image._encodedWidth;
1307 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001308 } else {
1309 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001310 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1311 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001312 }
1313 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001314 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001315 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001316 if (num_received_layers_ == num_expected_layers_) {
1317 encoded_frame_event_.Set();
1318 }
sprangb1ca0732017-02-01 08:38:12 -08001319 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001320 }
1321
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001322 void OnEncoderConfigurationChanged(
1323 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001324 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001325 VideoEncoderConfig::ContentType content_type,
1326 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001327 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001328 ++number_of_reconfigurations_;
1329 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1330 }
1331
Per Kjellanderdcef6412020-10-07 15:09:05 +02001332 void OnBitrateAllocationUpdated(
1333 const VideoBitrateAllocation& allocation) override {
1334 MutexLock lock(&mutex_);
1335 ++number_of_bitrate_allocations_;
1336 last_bitrate_allocation_ = allocation;
1337 }
1338
Per Kjellandera9434842020-10-15 17:53:22 +02001339 void OnVideoLayersAllocationUpdated(
1340 VideoLayersAllocation allocation) override {
1341 MutexLock lock(&mutex_);
1342 ++number_of_layers_allocations_;
1343 last_layers_allocation_ = allocation;
1344 rtc::StringBuilder log;
1345 for (const auto& layer : allocation.active_spatial_layers) {
1346 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1347 << "[";
1348 for (const auto target_bitrate :
1349 layer.target_bitrate_per_temporal_layer) {
1350 log << target_bitrate.kbps() << ",";
1351 }
1352 log << "]";
1353 }
1354 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1355 }
1356
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001357 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001358 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001359 TestEncoder* test_encoder_;
1360 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001361 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001362 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001363 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001364 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001365 uint32_t last_height_ = 0;
1366 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001367 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001368 size_t num_expected_layers_ = 1;
1369 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001370 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001371 int number_of_reconfigurations_ = 0;
1372 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001373 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1374 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001375 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1376 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001377 };
1378
Sergey Silkin5ee69672019-07-02 14:18:34 +02001379 class VideoBitrateAllocatorProxyFactory
1380 : public VideoBitrateAllocatorFactory {
1381 public:
1382 VideoBitrateAllocatorProxyFactory()
1383 : bitrate_allocator_factory_(
1384 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1385
1386 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1387 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001388 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001389 codec_config_ = codec;
1390 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1391 }
1392
1393 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001394 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001395 return codec_config_;
1396 }
1397
1398 private:
1399 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1400
Markus Handella3765182020-07-08 13:13:32 +02001401 mutable Mutex mutex_;
1402 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001403 };
1404
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001405 Clock* clock() { return time_controller_.GetClock(); }
1406 void AdvanceTime(TimeDelta duration) {
1407 time_controller_.AdvanceTime(duration);
1408 }
1409
1410 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1411
1412 protected:
1413 virtual TaskQueueFactory* GetTaskQueueFactory() {
1414 return time_controller_.GetTaskQueueFactory();
1415 }
1416
1417 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001418 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001419 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001420 int codec_width_;
1421 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001422 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001423 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001424 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001425 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001426 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001427 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001428 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001429 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001430};
1431
mflodmancc3d4422017-08-03 08:27:51 -07001432TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001434 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001435 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001436 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001437 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001438 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001439 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001440}
1441
mflodmancc3d4422017-08-03 08:27:51 -07001442TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001443 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001444 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001445 // The encoder will cache up to one frame for a short duration. Adding two
1446 // frames means that the first frame will be dropped and the second frame will
1447 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001448 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001449 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001450 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001451 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001452
Henrik Boström381d1092020-05-12 18:49:07 +02001453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001454 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001455
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001457 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001458 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1459
1460 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001461 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001462}
1463
mflodmancc3d4422017-08-03 08:27:51 -07001464TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001465 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001466 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001467 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001468 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001469
Henrik Boström381d1092020-05-12 18:49:07 +02001470 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001471 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1472
Sebastian Janssona3177052018-04-10 13:05:49 +02001473 // The encoder will cache up to one frame for a short duration. Adding two
1474 // frames means that the first frame will be dropped and the second frame will
1475 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001476 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001477 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001478
Henrik Boström381d1092020-05-12 18:49:07 +02001479 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001480 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001481 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001482 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1483 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001484 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001485}
1486
mflodmancc3d4422017-08-03 08:27:51 -07001487TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001489 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001490 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001491 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001492
1493 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001494 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001495
perkja49cbd32016-09-16 07:53:41 -07001496 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001497 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001498 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001499}
1500
mflodmancc3d4422017-08-03 08:27:51 -07001501TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001503 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001504
perkja49cbd32016-09-16 07:53:41 -07001505 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001506 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001507
mflodmancc3d4422017-08-03 08:27:51 -07001508 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001509 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001510 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001511 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1512 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001513}
1514
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001515class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1516 public:
1517 VideoStreamEncoderBlockedTest() {}
1518
1519 TaskQueueFactory* GetTaskQueueFactory() override {
1520 return task_queue_factory_.get();
1521 }
1522
1523 private:
1524 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1525 CreateDefaultTaskQueueFactory();
1526};
1527
1528TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001530 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001531
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001532 int dropped_count = 0;
1533 stats_proxy_->SetDroppedFrameCallback(
1534 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1535 ++dropped_count;
1536 });
1537
perkj26091b12016-09-01 01:17:40 -07001538 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001539 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001540 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001541 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1542 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001543 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1544 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001545 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001546 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001547
mflodmancc3d4422017-08-03 08:27:51 -07001548 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001549
1550 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001551}
1552
Henrik Boström56db9ff2021-03-24 09:06:45 +01001553TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001555 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001556
1557 rtc::Event frame_destroyed_event;
1558 video_source_.IncomingCapturedFrame(
1559 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001560 WaitForEncodedFrame(1);
1561 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1562 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001563 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1564 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001565 video_stream_encoder_->Stop();
1566}
1567
Henrik Boström56db9ff2021-03-24 09:06:45 +01001568TEST_F(VideoStreamEncoderTest,
1569 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001570 // Use the cropping factory.
1571 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001572 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001573 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1574 kMaxPayloadLength);
1575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1576
1577 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001579 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001580 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1581 WaitForEncodedFrame(1);
1582 // The encoder will have been configured once.
1583 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001584 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1585 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001586
1587 // Now send in a fake frame that needs to be cropped as the width/height
1588 // aren't divisible by 4 (see CreateEncoderStreams above).
1589 rtc::Event frame_destroyed_event;
1590 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1591 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001592 WaitForEncodedFrame(2);
1593 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1594 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001595 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1596 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001597 video_stream_encoder_->Stop();
1598}
1599
Evan Shrubsole895556e2020-10-05 09:15:13 +02001600TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001603
1604 video_source_.IncomingCapturedFrame(
1605 CreateNV12Frame(1, codec_width_, codec_height_));
1606 WaitForEncodedFrame(1);
1607 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1608 fake_encoder_.GetLastInputPixelFormat());
1609 video_stream_encoder_->Stop();
1610}
1611
Henrik Boström56db9ff2021-03-24 09:06:45 +01001612TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001614 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001615
1616 fake_encoder_.SetPreferredPixelFormats({});
1617
1618 rtc::Event frame_destroyed_event;
1619 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1620 1, &frame_destroyed_event, codec_width_, codec_height_));
1621 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001622 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001623 fake_encoder_.GetLastInputPixelFormat());
1624 video_stream_encoder_->Stop();
1625}
1626
Henrik Boström56db9ff2021-03-24 09:06:45 +01001627TEST_F(VideoStreamEncoderTest,
1628 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001630 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001631
1632 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1633
1634 rtc::Event frame_destroyed_event;
1635 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1636 1, &frame_destroyed_event, codec_width_, codec_height_));
1637 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001638 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001639 fake_encoder_.GetLastInputPixelFormat());
1640 video_stream_encoder_->Stop();
1641}
1642
Henrik Boström56db9ff2021-03-24 09:06:45 +01001643TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001646
1647 // Fake NV12 native frame does not allow mapping to I444.
1648 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1649
1650 rtc::Event frame_destroyed_event;
1651 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1652 1, &frame_destroyed_event, codec_width_, codec_height_));
1653 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001654 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001655 fake_encoder_.GetLastInputPixelFormat());
1656 video_stream_encoder_->Stop();
1657}
1658
Henrik Boström56db9ff2021-03-24 09:06:45 +01001659TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001661 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001662
1663 rtc::Event frame_destroyed_event;
1664 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1665 1, &frame_destroyed_event, codec_width_, codec_height_));
1666 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001667 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001668 fake_encoder_.GetLastInputPixelFormat());
1669 video_stream_encoder_->Stop();
1670}
1671
Ying Wang9b881ab2020-02-07 14:29:32 +01001672TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001674 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001675 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1676 WaitForEncodedFrame(1);
1677
Henrik Boström381d1092020-05-12 18:49:07 +02001678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001679 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001680 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1681 // frames. Adding two frames means that the first frame will be dropped and
1682 // the second frame will be sent to the encoder.
1683 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1684 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1685 WaitForEncodedFrame(3);
1686 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1687 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1688 WaitForEncodedFrame(5);
1689 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1690 video_stream_encoder_->Stop();
1691}
1692
mflodmancc3d4422017-08-03 08:27:51 -07001693TEST_F(VideoStreamEncoderTest,
1694 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001696 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001697 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001698
1699 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001700 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001701 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001702 // The encoder will have been configured once when the first frame is
1703 // received.
1704 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001705
1706 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001707 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001708 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001709 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001710 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001711
1712 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001713 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001714 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001715 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001716 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001717
mflodmancc3d4422017-08-03 08:27:51 -07001718 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001719}
1720
mflodmancc3d4422017-08-03 08:27:51 -07001721TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001722 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001723 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001724
1725 // Capture a frame and wait for it to synchronize with the encoder thread.
1726 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001727 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001728 // The encoder will have been configured once.
1729 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001730 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1731 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001732
1733 codec_width_ *= 2;
1734 codec_height_ *= 2;
1735 // Capture a frame with a higher resolution and wait for it to synchronize
1736 // with the encoder thread.
1737 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001738 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001739 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1740 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001741 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001742
mflodmancc3d4422017-08-03 08:27:51 -07001743 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001744}
1745
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001746TEST_F(VideoStreamEncoderTest,
1747 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001749 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001750
1751 // Capture a frame and wait for it to synchronize with the encoder thread.
1752 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1753 WaitForEncodedFrame(1);
1754
1755 VideoEncoderConfig video_encoder_config;
1756 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1757 // Changing the max payload data length recreates encoder.
1758 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1759 kMaxPayloadLength / 2);
1760
1761 // Capture a frame and wait for it to synchronize with the encoder thread.
1762 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1763 WaitForEncodedFrame(2);
1764 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1765
1766 video_stream_encoder_->Stop();
1767}
1768
Sergey Silkin5ee69672019-07-02 14:18:34 +02001769TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001771 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001772
1773 VideoEncoderConfig video_encoder_config;
1774 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1776 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001777 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1778 kMaxPayloadLength);
1779
1780 // Capture a frame and wait for it to synchronize with the encoder thread.
1781 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1782 WaitForEncodedFrame(1);
1783 // The encoder will have been configured once when the first frame is
1784 // received.
1785 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001786 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001787 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001788 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001789 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1790
Sergey Silkin6456e352019-07-08 17:56:40 +02001791 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1792 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001793 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1794 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001795 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1796 kMaxPayloadLength);
1797
1798 // Capture a frame and wait for it to synchronize with the encoder thread.
1799 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1800 WaitForEncodedFrame(2);
1801 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1802 // Bitrate limits have changed - rate allocator should be reconfigured,
1803 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001804 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001805 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001806 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001807 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001808 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001809
1810 video_stream_encoder_->Stop();
1811}
1812
Sergey Silkin6456e352019-07-08 17:56:40 +02001813TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001814 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001816 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001817
Sergey Silkincd02eba2020-01-20 14:48:40 +01001818 const uint32_t kMinEncBitrateKbps = 100;
1819 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001820 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001821 /*frame_size_pixels=*/codec_width_ * codec_height_,
1822 /*min_start_bitrate_bps=*/0,
1823 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1824 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001825 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1826
Sergey Silkincd02eba2020-01-20 14:48:40 +01001827 VideoEncoderConfig video_encoder_config;
1828 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1829 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1830 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1831 (kMinEncBitrateKbps + 1) * 1000;
1832 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1833 kMaxPayloadLength);
1834
1835 // When both encoder and app provide bitrate limits, the intersection of
1836 // provided sets should be used.
1837 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1838 WaitForEncodedFrame(1);
1839 EXPECT_EQ(kMaxEncBitrateKbps,
1840 bitrate_allocator_factory_.codec_config().maxBitrate);
1841 EXPECT_EQ(kMinEncBitrateKbps + 1,
1842 bitrate_allocator_factory_.codec_config().minBitrate);
1843
1844 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1845 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1846 (kMinEncBitrateKbps - 1) * 1000;
1847 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1848 kMaxPayloadLength);
1849 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001850 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001851 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001852 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001853 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001854 bitrate_allocator_factory_.codec_config().minBitrate);
1855
Sergey Silkincd02eba2020-01-20 14:48:40 +01001856 video_stream_encoder_->Stop();
1857}
1858
1859TEST_F(VideoStreamEncoderTest,
1860 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001861 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001862 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001863
1864 const uint32_t kMinAppBitrateKbps = 100;
1865 const uint32_t kMaxAppBitrateKbps = 200;
1866 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1867 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1868 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1869 /*frame_size_pixels=*/codec_width_ * codec_height_,
1870 /*min_start_bitrate_bps=*/0,
1871 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1872 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1873 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1874
1875 VideoEncoderConfig video_encoder_config;
1876 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1877 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1878 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1879 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001880 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1881 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001882
Sergey Silkincd02eba2020-01-20 14:48:40 +01001883 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1884 WaitForEncodedFrame(1);
1885 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001886 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001887 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001888 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001889
1890 video_stream_encoder_->Stop();
1891}
1892
1893TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001894 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001897
1898 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001899 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001900 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001901 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001902 fake_encoder_.SetResolutionBitrateLimits(
1903 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1904
1905 VideoEncoderConfig video_encoder_config;
1906 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1907 video_encoder_config.max_bitrate_bps = 0;
1908 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1909 kMaxPayloadLength);
1910
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001911 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001912 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1913 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001914 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1915 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001916 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1917 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1918
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001920 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1921 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001922 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1923 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001924 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1925 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1926
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001927 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001928 // encoder for 360p should be used.
1929 video_source_.IncomingCapturedFrame(
1930 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1931 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1933 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001934 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1935 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1936
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001937 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001938 // ignored.
1939 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1940 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001941 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1942 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001943 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1944 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001945 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1946 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001947 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1948 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1949
1950 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1951 // for 270p should be used.
1952 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1953 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001954 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1955 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001956 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1957 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1958
1959 video_stream_encoder_->Stop();
1960}
1961
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001964 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001965
1966 VideoEncoderConfig video_encoder_config;
1967 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1968 video_encoder_config.max_bitrate_bps = 0;
1969 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1970 kMaxPayloadLength);
1971
1972 // Encode 720p frame to get the default encoder target bitrate.
1973 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1974 WaitForEncodedFrame(1);
1975 const uint32_t kDefaultTargetBitrateFor720pKbps =
1976 bitrate_allocator_factory_.codec_config()
1977 .simulcastStream[0]
1978 .targetBitrate;
1979
1980 // Set the max recommended encoder bitrate to something lower than the default
1981 // target bitrate.
1982 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1983 1280 * 720, 10 * 1000, 10 * 1000,
1984 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1985 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1986
1987 // Change resolution to trigger encoder reinitialization.
1988 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1989 WaitForEncodedFrame(2);
1990 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1991 WaitForEncodedFrame(3);
1992
1993 // Ensure the target bitrate is capped by the max bitrate.
1994 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1995 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1996 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1997 .simulcastStream[0]
1998 .targetBitrate *
1999 1000,
2000 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2001
2002 video_stream_encoder_->Stop();
2003}
2004
Åsa Perssona7e34d32021-01-20 15:36:13 +01002005TEST_F(VideoStreamEncoderTest,
2006 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2007 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2008 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2009 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2010 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2011 fake_encoder_.SetResolutionBitrateLimits(
2012 {kEncoderLimits270p, kEncoderLimits360p});
2013
2014 // Two streams, highest stream active.
2015 VideoEncoderConfig config;
2016 const int kNumStreams = 2;
2017 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2018 config.max_bitrate_bps = 0;
2019 config.simulcast_layers[0].active = false;
2020 config.simulcast_layers[1].active = true;
2021 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002022 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002023 "VP8", /*max qp*/ 56, /*screencast*/ false,
2024 /*screenshare enabled*/ false);
2025 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2026
2027 // The encoder bitrate limits for 270p should be used.
2028 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2029 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002030 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002031 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002032 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002033 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002034 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002035
2036 // The encoder bitrate limits for 360p should be used.
2037 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2038 EXPECT_FALSE(WaitForFrame(1000));
2039 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002040 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002041 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002042 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002043
2044 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2045 video_source_.IncomingCapturedFrame(
2046 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2047 EXPECT_FALSE(WaitForFrame(1000));
2048 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002049 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002050 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002051 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002052
2053 // Resolution higher than 360p. Encoder limits should be ignored.
2054 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2055 EXPECT_FALSE(WaitForFrame(1000));
2056 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002057 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002058 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002059 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002060 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002061 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002062 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002063 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002064
2065 // Resolution lower than 270p. The encoder limits for 270p should be used.
2066 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2067 EXPECT_FALSE(WaitForFrame(1000));
2068 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002069 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002070 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002071 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002072
2073 video_stream_encoder_->Stop();
2074}
2075
2076TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002077 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2078 // Two streams, highest stream active.
2079 VideoEncoderConfig config;
2080 const int kNumStreams = 2;
2081 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2082 config.max_bitrate_bps = 0;
2083 config.simulcast_layers[0].active = false;
2084 config.simulcast_layers[1].active = true;
2085 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002086 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002087 "VP8", /*max qp*/ 56, /*screencast*/ false,
2088 /*screenshare enabled*/ false);
2089 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2090
2091 // Default bitrate limits for 270p should be used.
2092 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2093 kDefaultLimits270p =
2094 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002095 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002096 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2097 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002098 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002099 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002100 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002101 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002102 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002103
2104 // Default bitrate limits for 360p should be used.
2105 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2106 kDefaultLimits360p =
2107 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002108 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002109 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2110 EXPECT_FALSE(WaitForFrame(1000));
2111 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002112 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002113 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002114 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002115
2116 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2117 video_source_.IncomingCapturedFrame(
2118 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2119 EXPECT_FALSE(WaitForFrame(1000));
2120 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002121 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002122 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002123 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002124
2125 // Default bitrate limits for 540p should be used.
2126 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2127 kDefaultLimits540p =
2128 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002129 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002130 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2131 EXPECT_FALSE(WaitForFrame(1000));
2132 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002133 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002134 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002135 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002136
2137 video_stream_encoder_->Stop();
2138}
2139
2140TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002141 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2142 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2143 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2144 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2145 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2146 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2147 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2148 fake_encoder_.SetResolutionBitrateLimits(
2149 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2150
2151 // Three streams, middle stream active.
2152 VideoEncoderConfig config;
2153 const int kNumStreams = 3;
2154 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2155 config.simulcast_layers[0].active = false;
2156 config.simulcast_layers[1].active = true;
2157 config.simulcast_layers[2].active = false;
2158 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002159 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002160 "VP8", /*max qp*/ 56, /*screencast*/ false,
2161 /*screenshare enabled*/ false);
2162 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2163
2164 // The encoder bitrate limits for 360p should be used.
2165 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2166 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002167 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002168 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002169 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002170 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002171 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002172
2173 // The encoder bitrate limits for 270p should be used.
2174 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002177 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002178 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002179 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002180
2181 video_stream_encoder_->Stop();
2182}
2183
2184TEST_F(VideoStreamEncoderTest,
2185 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2186 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2187 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2188 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2189 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2190 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2191 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2192 fake_encoder_.SetResolutionBitrateLimits(
2193 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2194
2195 // Three streams, lowest stream active.
2196 VideoEncoderConfig config;
2197 const int kNumStreams = 3;
2198 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2199 config.simulcast_layers[0].active = true;
2200 config.simulcast_layers[1].active = false;
2201 config.simulcast_layers[2].active = false;
2202 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002203 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 "VP8", /*max qp*/ 56, /*screencast*/ false,
2205 /*screenshare enabled*/ false);
2206 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2207
2208 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2209 // on lowest stream, limits for 270p should not be used
2210 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2211 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002212 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002213 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002214 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002216 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002217
2218 video_stream_encoder_->Stop();
2219}
2220
2221TEST_F(VideoStreamEncoderTest,
2222 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2223 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2224 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2225 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2226 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2227 fake_encoder_.SetResolutionBitrateLimits(
2228 {kEncoderLimits270p, kEncoderLimits360p});
2229 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2230
2231 // Two streams, highest stream active.
2232 VideoEncoderConfig config;
2233 const int kNumStreams = 2;
2234 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2235 config.simulcast_layers[0].active = false;
2236 config.simulcast_layers[1].active = true;
2237 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2238 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002239 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240 "VP8", /*max qp*/ 56, /*screencast*/ false,
2241 /*screenshare enabled*/ false);
2242 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2243
2244 // The encoder bitrate limits for 270p should be used.
2245 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2246 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002248 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002249 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002250 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002251 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002252
2253 // The max configured bitrate is less than the encoder limit for 360p.
2254 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2255 EXPECT_FALSE(WaitForFrame(1000));
2256 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002257 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002258 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002259 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002260
2261 video_stream_encoder_->Stop();
2262}
2263
mflodmancc3d4422017-08-03 08:27:51 -07002264TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002265 EXPECT_TRUE(video_source_.has_sinks());
2266 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002268 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002269 EXPECT_FALSE(video_source_.has_sinks());
2270 EXPECT_TRUE(new_video_source.has_sinks());
2271
mflodmancc3d4422017-08-03 08:27:51 -07002272 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002273}
2274
mflodmancc3d4422017-08-03 08:27:51 -07002275TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002276 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002277 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002278 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002279 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002280}
2281
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002282class ResolutionAlignmentTest
2283 : public VideoStreamEncoderTest,
2284 public ::testing::WithParamInterface<
2285 ::testing::tuple<int, std::vector<double>>> {
2286 public:
2287 ResolutionAlignmentTest()
2288 : requested_alignment_(::testing::get<0>(GetParam())),
2289 scale_factors_(::testing::get<1>(GetParam())) {}
2290
2291 protected:
2292 const int requested_alignment_;
2293 const std::vector<double> scale_factors_;
2294};
2295
2296INSTANTIATE_TEST_SUITE_P(
2297 AlignmentAndScaleFactors,
2298 ResolutionAlignmentTest,
2299 ::testing::Combine(
2300 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2301 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2302 std::vector<double>{-1.0, -1.0},
2303 std::vector<double>{-1.0, -1.0, -1.0},
2304 std::vector<double>{4.0, 2.0, 1.0},
2305 std::vector<double>{9999.0, -1.0, 1.0},
2306 std::vector<double>{3.99, 2.01, 1.0},
2307 std::vector<double>{4.9, 1.7, 1.25},
2308 std::vector<double>{10.0, 4.0, 3.0},
2309 std::vector<double>{1.75, 3.5},
2310 std::vector<double>{1.5, 2.5},
2311 std::vector<double>{1.3, 1.0})));
2312
2313TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2314 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002315 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002316 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2317 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2318
2319 // Fill config with the scaling factor by which to reduce encoding size.
2320 const int num_streams = scale_factors_.size();
2321 VideoEncoderConfig config;
2322 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2323 for (int i = 0; i < num_streams; ++i) {
2324 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2325 }
2326 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002327 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002328 "VP8", /*max qp*/ 56, /*screencast*/ false,
2329 /*screenshare enabled*/ false);
2330 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2331
Henrik Boström381d1092020-05-12 18:49:07 +02002332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002333 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2334 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002335 // Wait for all layers before triggering event.
2336 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002337
2338 // On the 1st frame, we should have initialized the encoder and
2339 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002340 int64_t timestamp_ms = kFrameIntervalMs;
2341 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2342 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002343 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002344
2345 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2346 // (It's up the to the encoder to potentially drop the previous frame,
2347 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002348 timestamp_ms += kFrameIntervalMs;
2349 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2350 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002351 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002352
Asa Persson606d3cb2021-10-04 10:07:11 +02002353 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002354 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2355 // Frame size should be a multiple of the requested alignment.
2356 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2357 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2358 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2359 // Aspect ratio should match.
2360 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2361 codec.height * codec.simulcastStream[i].width);
2362 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002363
2364 video_stream_encoder_->Stop();
2365}
2366
Jonathan Yubc771b72017-12-08 17:04:29 -08002367TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2368 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002369 const int kWidth = 1280;
2370 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002371
2372 // We rely on the automatic resolution adaptation, but we handle framerate
2373 // adaptation manually by mocking the stats proxy.
2374 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002375
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002376 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002378 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002379 video_stream_encoder_->SetSource(&video_source_,
2380 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002381 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002382 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002383 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002384 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2385
Jonathan Yubc771b72017-12-08 17:04:29 -08002386 // Adapt down as far as possible.
2387 rtc::VideoSinkWants last_wants;
2388 int64_t t = 1;
2389 int loop_count = 0;
2390 do {
2391 ++loop_count;
2392 last_wants = video_source_.sink_wants();
2393
2394 // Simulate the framerate we've been asked to adapt to.
2395 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2396 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2397 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2398 mock_stats.input_frame_rate = fps;
2399 stats_proxy_->SetMockStats(mock_stats);
2400
2401 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2402 sink_.WaitForEncodedFrame(t);
2403 t += frame_interval_ms;
2404
mflodmancc3d4422017-08-03 08:27:51 -07002405 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002406 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002407 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002408 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2409 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002410 } while (video_source_.sink_wants().max_pixel_count <
2411 last_wants.max_pixel_count ||
2412 video_source_.sink_wants().max_framerate_fps <
2413 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002414
Jonathan Yubc771b72017-12-08 17:04:29 -08002415 // Verify that we've adapted all the way down.
2416 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002417 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002418 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2419 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002420 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002421 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2422 *video_source_.last_sent_height());
2423 EXPECT_EQ(kMinBalancedFramerateFps,
2424 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002425
Jonathan Yubc771b72017-12-08 17:04:29 -08002426 // Adapt back up the same number of times we adapted down.
2427 for (int i = 0; i < loop_count - 1; ++i) {
2428 last_wants = video_source_.sink_wants();
2429
2430 // Simulate the framerate we've been asked to adapt to.
2431 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2432 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2433 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2434 mock_stats.input_frame_rate = fps;
2435 stats_proxy_->SetMockStats(mock_stats);
2436
2437 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2438 sink_.WaitForEncodedFrame(t);
2439 t += frame_interval_ms;
2440
Henrik Boström91aa7322020-04-28 12:24:33 +02002441 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002442 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002443 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002444 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2445 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002446 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2447 last_wants.max_pixel_count ||
2448 video_source_.sink_wants().max_framerate_fps >
2449 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002450 }
2451
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002452 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002453 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002454 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002455 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2456 EXPECT_EQ((loop_count - 1) * 2,
2457 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002458
mflodmancc3d4422017-08-03 08:27:51 -07002459 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002460}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002461
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002462TEST_F(VideoStreamEncoderTest,
2463 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002464 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2465 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002466 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002467
2468 const int kFrameWidth = 1280;
2469 const int kFrameHeight = 720;
2470
2471 int64_t ntp_time = kFrameIntervalMs;
2472
2473 // Force an input frame rate to be available, or the adaptation call won't
2474 // know what framerate to adapt form.
2475 const int kInputFps = 30;
2476 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2477 stats.input_frame_rate = kInputFps;
2478 stats_proxy_->SetMockStats(stats);
2479
2480 video_source_.set_adaptation_enabled(true);
2481 video_stream_encoder_->SetSource(
2482 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002483 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002484 video_source_.IncomingCapturedFrame(
2485 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2486 sink_.WaitForEncodedFrame(ntp_time);
2487 ntp_time += kFrameIntervalMs;
2488
2489 // Trigger CPU overuse.
2490 video_stream_encoder_->TriggerCpuOveruse();
2491 video_source_.IncomingCapturedFrame(
2492 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2493 sink_.WaitForEncodedFrame(ntp_time);
2494 ntp_time += kFrameIntervalMs;
2495
2496 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2497 EXPECT_EQ(std::numeric_limits<int>::max(),
2498 video_source_.sink_wants().max_pixel_count);
2499 // Some framerate constraint should be set.
2500 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2501 EXPECT_LT(restricted_fps, kInputFps);
2502 video_source_.IncomingCapturedFrame(
2503 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2504 sink_.WaitForEncodedFrame(ntp_time);
2505 ntp_time += 100;
2506
Henrik Boström2671dac2020-05-19 16:29:09 +02002507 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002508 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2509 // Give the encoder queue time to process the change in degradation preference
2510 // by waiting for an encoded frame.
2511 video_source_.IncomingCapturedFrame(
2512 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2513 sink_.WaitForEncodedFrame(ntp_time);
2514 ntp_time += kFrameIntervalMs;
2515
2516 video_stream_encoder_->TriggerQualityLow();
2517 video_source_.IncomingCapturedFrame(
2518 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2519 sink_.WaitForEncodedFrame(ntp_time);
2520 ntp_time += kFrameIntervalMs;
2521
2522 // Some resolution constraint should be set.
2523 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2524 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2525 kFrameWidth * kFrameHeight);
2526 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2527
2528 int pixel_count = video_source_.sink_wants().max_pixel_count;
2529 // Triggering a CPU underuse should not change the sink wants since it has
2530 // not been overused for resolution since we changed degradation preference.
2531 video_stream_encoder_->TriggerCpuUnderuse();
2532 video_source_.IncomingCapturedFrame(
2533 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2534 sink_.WaitForEncodedFrame(ntp_time);
2535 ntp_time += kFrameIntervalMs;
2536 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2537 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2538
Evan Shrubsole64469032020-06-11 10:45:29 +02002539 // Change the degradation preference back. CPU underuse should not adapt since
2540 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002541 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002542 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2543 video_source_.IncomingCapturedFrame(
2544 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2545 sink_.WaitForEncodedFrame(ntp_time);
2546 ntp_time += 100;
2547 // Resolution adaptations is gone after changing degradation preference.
2548 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2549 EXPECT_EQ(std::numeric_limits<int>::max(),
2550 video_source_.sink_wants().max_pixel_count);
2551 // The fps adaptation from above is now back.
2552 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2553
2554 // Trigger CPU underuse.
2555 video_stream_encoder_->TriggerCpuUnderuse();
2556 video_source_.IncomingCapturedFrame(
2557 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2558 sink_.WaitForEncodedFrame(ntp_time);
2559 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002560 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2561
2562 // Trigger QP underuse, fps should return to normal.
2563 video_stream_encoder_->TriggerQualityHigh();
2564 video_source_.IncomingCapturedFrame(
2565 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2566 sink_.WaitForEncodedFrame(ntp_time);
2567 ntp_time += kFrameIntervalMs;
2568 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002569
2570 video_stream_encoder_->Stop();
2571}
2572
mflodmancc3d4422017-08-03 08:27:51 -07002573TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002575 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002576 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002577
sprangc5d62e22017-04-02 23:53:04 -07002578 const int kFrameWidth = 1280;
2579 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002580
Åsa Persson8c1bf952018-09-13 10:42:19 +02002581 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002582
kthelgason5e13d412016-12-01 03:59:51 -08002583 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002584 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002585 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002586 frame_timestamp += kFrameIntervalMs;
2587
perkj803d97f2016-11-01 11:45:46 -07002588 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002589 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002590 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002591 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002592 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002593 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002594
asapersson0944a802017-04-07 00:57:58 -07002595 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002596 // wanted resolution.
2597 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2598 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2599 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002600 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002601
2602 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002603 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002604 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002605 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002606 // Give the encoder queue time to process the change in degradation preference
2607 // by waiting for an encoded frame.
2608 new_video_source.IncomingCapturedFrame(
2609 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2610 sink_.WaitForEncodedFrame(frame_timestamp);
2611 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002612 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002613 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002614
sprangc5d62e22017-04-02 23:53:04 -07002615 // Force an input frame rate to be available, or the adaptation call won't
2616 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002617 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002618 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002619 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002620 stats_proxy_->SetMockStats(stats);
2621
mflodmancc3d4422017-08-03 08:27:51 -07002622 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002623 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002624 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002625 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002626 frame_timestamp += kFrameIntervalMs;
2627
2628 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002629 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002630 EXPECT_EQ(std::numeric_limits<int>::max(),
2631 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002632 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002633
asapersson02465b82017-04-10 01:12:52 -07002634 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002635 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2636 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002637 // Give the encoder queue time to process the change in degradation preference
2638 // by waiting for an encoded frame.
2639 new_video_source.IncomingCapturedFrame(
2640 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2641 sink_.WaitForEncodedFrame(frame_timestamp);
2642 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002643 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002644
mflodmancc3d4422017-08-03 08:27:51 -07002645 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002646 new_video_source.IncomingCapturedFrame(
2647 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002648 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002649 frame_timestamp += kFrameIntervalMs;
2650
2651 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002652 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002653
2654 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002655 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002656 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002657 // Give the encoder queue time to process the change in degradation preference
2658 // by waiting for an encoded frame.
2659 new_video_source.IncomingCapturedFrame(
2660 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2661 sink_.WaitForEncodedFrame(frame_timestamp);
2662 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002663 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2664 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002665 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002666 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002667
2668 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002669 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002670 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002671 // Give the encoder queue time to process the change in degradation preference
2672 // by waiting for an encoded frame.
2673 new_video_source.IncomingCapturedFrame(
2674 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2675 sink_.WaitForEncodedFrame(frame_timestamp);
2676 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002677 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2678 EXPECT_EQ(std::numeric_limits<int>::max(),
2679 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002680 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002681
mflodmancc3d4422017-08-03 08:27:51 -07002682 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002683}
2684
mflodmancc3d4422017-08-03 08:27:51 -07002685TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002687 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002688
asaperssonfab67072017-04-04 05:51:49 -07002689 const int kWidth = 1280;
2690 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002691 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002692 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002693 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2694 EXPECT_FALSE(stats.bw_limited_resolution);
2695 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2696
2697 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002698 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002699 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002700 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002701
2702 stats = stats_proxy_->GetStats();
2703 EXPECT_TRUE(stats.bw_limited_resolution);
2704 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2705
2706 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002707 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002708 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002709 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002710
2711 stats = stats_proxy_->GetStats();
2712 EXPECT_FALSE(stats.bw_limited_resolution);
2713 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2714 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2715
mflodmancc3d4422017-08-03 08:27:51 -07002716 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002717}
2718
mflodmancc3d4422017-08-03 08:27:51 -07002719TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002720 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002721 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002722
2723 const int kWidth = 1280;
2724 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002725 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002726 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002727 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2728 EXPECT_FALSE(stats.cpu_limited_resolution);
2729 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2730
2731 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002732 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002733 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002734 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002735
2736 stats = stats_proxy_->GetStats();
2737 EXPECT_TRUE(stats.cpu_limited_resolution);
2738 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2739
2740 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002741 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002742 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002743 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002744
2745 stats = stats_proxy_->GetStats();
2746 EXPECT_FALSE(stats.cpu_limited_resolution);
2747 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002748 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002749
mflodmancc3d4422017-08-03 08:27:51 -07002750 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002751}
2752
mflodmancc3d4422017-08-03 08:27:51 -07002753TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002754 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002755 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002756
asaperssonfab67072017-04-04 05:51:49 -07002757 const int kWidth = 1280;
2758 const int kHeight = 720;
2759 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002760 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002761 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002762 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002763 EXPECT_FALSE(stats.cpu_limited_resolution);
2764 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2765
asaperssonfab67072017-04-04 05:51:49 -07002766 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002767 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002768 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002769 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002770 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002771 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002772 EXPECT_TRUE(stats.cpu_limited_resolution);
2773 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2774
2775 // Set new source with adaptation still enabled.
2776 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002777 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002779
asaperssonfab67072017-04-04 05:51:49 -07002780 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002781 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002782 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002783 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002784 EXPECT_TRUE(stats.cpu_limited_resolution);
2785 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2786
2787 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002788 video_stream_encoder_->SetSource(&new_video_source,
2789 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002790
asaperssonfab67072017-04-04 05:51:49 -07002791 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002792 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002793 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002794 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002795 EXPECT_FALSE(stats.cpu_limited_resolution);
2796 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2797
2798 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002800 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002801
asaperssonfab67072017-04-04 05:51:49 -07002802 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002803 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002804 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002805 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002806 EXPECT_TRUE(stats.cpu_limited_resolution);
2807 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2808
asaperssonfab67072017-04-04 05:51:49 -07002809 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002810 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002811 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002812 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002813 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002814 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002815 EXPECT_FALSE(stats.cpu_limited_resolution);
2816 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002817 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002818
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002820}
2821
mflodmancc3d4422017-08-03 08:27:51 -07002822TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002825
asaperssonfab67072017-04-04 05:51:49 -07002826 const int kWidth = 1280;
2827 const int kHeight = 720;
2828 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002829 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002830 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002831 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002832 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002833 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002834
2835 // Set new source with adaptation still enabled.
2836 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002837 video_stream_encoder_->SetSource(&new_video_source,
2838 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002839
asaperssonfab67072017-04-04 05:51:49 -07002840 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002841 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002842 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002843 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002844 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002845 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002846
asaperssonfab67072017-04-04 05:51:49 -07002847 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002848 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002849 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002850 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002851 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002852 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002853 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002854 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002855
asaperssonfab67072017-04-04 05:51:49 -07002856 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002857 video_stream_encoder_->SetSource(&new_video_source,
2858 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002859
asaperssonfab67072017-04-04 05:51:49 -07002860 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002861 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002862 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002863 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002864 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002865 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002866
asapersson02465b82017-04-10 01:12:52 -07002867 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002868 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002869 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002870
asaperssonfab67072017-04-04 05:51:49 -07002871 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002872 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002873 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002874 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002875 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002876 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2877 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002878
mflodmancc3d4422017-08-03 08:27:51 -07002879 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002880}
2881
mflodmancc3d4422017-08-03 08:27:51 -07002882TEST_F(VideoStreamEncoderTest,
2883 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002886
2887 const int kWidth = 1280;
2888 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002889 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002890 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002891 video_source_.IncomingCapturedFrame(
2892 CreateFrame(timestamp_ms, kWidth, kHeight));
2893 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002894 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2895 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2896 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2897
2898 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002899 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002900 timestamp_ms += kFrameIntervalMs;
2901 video_source_.IncomingCapturedFrame(
2902 CreateFrame(timestamp_ms, kWidth, kHeight));
2903 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002904 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2906 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2907
2908 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002910 timestamp_ms += kFrameIntervalMs;
2911 video_source_.IncomingCapturedFrame(
2912 CreateFrame(timestamp_ms, kWidth, kHeight));
2913 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002914 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2915 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2916 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2917
Niels Möller4db138e2018-04-19 09:04:13 +02002918 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002919 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002920
2921 VideoEncoderConfig video_encoder_config;
2922 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2923 // Make format different, to force recreation of encoder.
2924 video_encoder_config.video_format.parameters["foo"] = "foo";
2925 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002926 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002927 timestamp_ms += kFrameIntervalMs;
2928 video_source_.IncomingCapturedFrame(
2929 CreateFrame(timestamp_ms, kWidth, kHeight));
2930 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002931 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2932 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2933 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2934
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002936}
2937
mflodmancc3d4422017-08-03 08:27:51 -07002938TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002939 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002940 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002941 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002942
2943 const int kWidth = 1280;
2944 const int kHeight = 720;
2945 int sequence = 1;
2946
2947 // Enable BALANCED preference, no initial limitation.
2948 test::FrameForwarder source;
2949 video_stream_encoder_->SetSource(&source,
2950 webrtc::DegradationPreference::BALANCED);
2951 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2952 WaitForEncodedFrame(sequence++);
2953 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2954 EXPECT_FALSE(stats.cpu_limited_resolution);
2955 EXPECT_FALSE(stats.cpu_limited_framerate);
2956 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2957
2958 // Trigger CPU overuse, should now adapt down.
2959 video_stream_encoder_->TriggerCpuOveruse();
2960 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2961 WaitForEncodedFrame(sequence++);
2962 stats = stats_proxy_->GetStats();
2963 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2964
2965 // Set new degradation preference should clear restrictions since we changed
2966 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002967 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002968 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2969 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2970 WaitForEncodedFrame(sequence++);
2971 stats = stats_proxy_->GetStats();
2972 EXPECT_FALSE(stats.cpu_limited_resolution);
2973 EXPECT_FALSE(stats.cpu_limited_framerate);
2974 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2975
2976 // Force an input frame rate to be available, or the adaptation call won't
2977 // know what framerate to adapt from.
2978 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2979 mock_stats.input_frame_rate = 30;
2980 stats_proxy_->SetMockStats(mock_stats);
2981 video_stream_encoder_->TriggerCpuOveruse();
2982 stats_proxy_->ResetMockStats();
2983 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2984 WaitForEncodedFrame(sequence++);
2985
2986 // We have now adapted once.
2987 stats = stats_proxy_->GetStats();
2988 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2989
2990 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002991 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2992 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002993 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2994 WaitForEncodedFrame(sequence++);
2995 stats = stats_proxy_->GetStats();
2996 EXPECT_FALSE(stats.cpu_limited_resolution);
2997 EXPECT_FALSE(stats.cpu_limited_framerate);
2998 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2999
3000 video_stream_encoder_->Stop();
3001}
3002
3003TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003004 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003006 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003007
asapersson0944a802017-04-07 00:57:58 -07003008 const int kWidth = 1280;
3009 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003010 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003011
asaperssonfab67072017-04-04 05:51:49 -07003012 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003013 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003014 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003015 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003016 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003017 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3018
asapersson02465b82017-04-10 01:12:52 -07003019 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003020 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003021 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003022 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003023 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003024 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003025 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003026 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3027
3028 // Set new source with adaptation still enabled.
3029 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003030 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003031 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003032
3033 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003034 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003035 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003036 stats = stats_proxy_->GetStats();
3037 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003038 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003039 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3040
sprangc5d62e22017-04-02 23:53:04 -07003041 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003042 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003043 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003044 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003045 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003046 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003047 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003048 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003049 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003050 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003051 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3052
sprangc5d62e22017-04-02 23:53:04 -07003053 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003054 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003055 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3056 mock_stats.input_frame_rate = 30;
3057 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003058 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003059 stats_proxy_->ResetMockStats();
3060
3061 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003062 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003063 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003064
3065 // Framerate now adapted.
3066 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003067 EXPECT_FALSE(stats.cpu_limited_resolution);
3068 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003069 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3070
3071 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003072 video_stream_encoder_->SetSource(&new_video_source,
3073 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003074 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003075 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003076 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003077
3078 stats = stats_proxy_->GetStats();
3079 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003080 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003081 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3082
3083 // Try to trigger overuse. Should not succeed.
3084 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003085 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003086 stats_proxy_->ResetMockStats();
3087
3088 stats = stats_proxy_->GetStats();
3089 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003090 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003091 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3092
3093 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003094 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003095 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003096 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003097 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003098 stats = stats_proxy_->GetStats();
3099 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003100 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003101 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003102
3103 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003104 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003105 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003106 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003107 stats = stats_proxy_->GetStats();
3108 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003109 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003110 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3111
3112 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003114 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003115 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003116 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003117 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003118 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003119 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003120 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003121 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003122 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3123
3124 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003125 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003126 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003127 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003128 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003129 stats = stats_proxy_->GetStats();
3130 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003131 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003132 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003133 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003134
mflodmancc3d4422017-08-03 08:27:51 -07003135 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003136}
3137
mflodmancc3d4422017-08-03 08:27:51 -07003138TEST_F(VideoStreamEncoderTest,
3139 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003140 const int kWidth = 1280;
3141 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003143 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003144
asaperssonfab67072017-04-04 05:51:49 -07003145 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003146 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003147
asaperssonfab67072017-04-04 05:51:49 -07003148 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003149 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003150
asaperssonfab67072017-04-04 05:51:49 -07003151 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003153
asaperssonfab67072017-04-04 05:51:49 -07003154 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003155 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003156
kthelgason876222f2016-11-29 01:44:11 -08003157 // Expect a scale down.
3158 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003159 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003160
asapersson02465b82017-04-10 01:12:52 -07003161 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003162 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003163 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003164 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003165
asaperssonfab67072017-04-04 05:51:49 -07003166 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003167 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003168 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003169 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003170
asaperssonfab67072017-04-04 05:51:49 -07003171 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003172 EXPECT_EQ(std::numeric_limits<int>::max(),
3173 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003174
asaperssonfab67072017-04-04 05:51:49 -07003175 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003176 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003177 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003178 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003179
asapersson02465b82017-04-10 01:12:52 -07003180 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003181 EXPECT_EQ(std::numeric_limits<int>::max(),
3182 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003183
mflodmancc3d4422017-08-03 08:27:51 -07003184 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003185}
3186
mflodmancc3d4422017-08-03 08:27:51 -07003187TEST_F(VideoStreamEncoderTest,
3188 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003189 const int kWidth = 1280;
3190 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003191 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003192 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003193
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003194 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003195 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003196 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003197 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003198
3199 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003200 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003201 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003202 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3203 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3204
3205 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003206 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003207 EXPECT_THAT(source.sink_wants(),
3208 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003209 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3210 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3211 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3212
3213 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003214 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003215 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3216 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3217 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3218
mflodmancc3d4422017-08-03 08:27:51 -07003219 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003220}
3221
mflodmancc3d4422017-08-03 08:27:51 -07003222TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003223 const int kWidth = 1280;
3224 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003225 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003226 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003227
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003228 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003229 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003230 video_stream_encoder_->SetSource(&source,
3231 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003232 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3233 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003234 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003235
3236 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003237 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003238 EXPECT_THAT(source.sink_wants(),
3239 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003240 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3241 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3242 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3243
3244 // Trigger adapt down for same input resolution, expect no change.
3245 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3246 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003247 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003248 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3249 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3250 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3251
3252 // Trigger adapt down for larger input resolution, expect no change.
3253 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3254 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003256 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3257 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3258 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3259
mflodmancc3d4422017-08-03 08:27:51 -07003260 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003261}
3262
mflodmancc3d4422017-08-03 08:27:51 -07003263TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003264 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3265 const int kWidth = 640;
3266 const int kHeight = 360;
3267 const int64_t kFrameIntervalMs = 150;
3268 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003269 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003270
3271 // Enable BALANCED preference, no initial limitation.
3272 AdaptingFrameForwarder source(&time_controller_);
3273 source.set_adaptation_enabled(true);
3274 video_stream_encoder_->SetSource(&source,
3275 webrtc::DegradationPreference::BALANCED);
3276
3277 int64_t timestamp_ms = kFrameIntervalMs;
3278 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3279 sink_.WaitForEncodedFrame(kWidth, kHeight);
3280 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3281 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3282 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3283 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3284
3285 // Trigger adapt down, expect reduced fps (640x360@15fps).
3286 video_stream_encoder_->TriggerQualityLow();
3287 timestamp_ms += kFrameIntervalMs;
3288 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3289 sink_.WaitForEncodedFrame(timestamp_ms);
3290 EXPECT_THAT(source.sink_wants(),
3291 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3293 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3294 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3295
3296 // Source requests 270p, expect reduced resolution (480x270@15fps).
3297 source.OnOutputFormatRequest(480, 270);
3298 timestamp_ms += kFrameIntervalMs;
3299 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3300 WaitForEncodedFrame(480, 270);
3301 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3302
3303 // Trigger adapt down, expect reduced fps (480x270@10fps).
3304 video_stream_encoder_->TriggerQualityLow();
3305 timestamp_ms += kFrameIntervalMs;
3306 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3307 sink_.WaitForEncodedFrame(timestamp_ms);
3308 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3309 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3310 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3311 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3312
3313 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3314 source.OnOutputFormatRequest(320, 180);
3315 timestamp_ms += kFrameIntervalMs;
3316 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3317 WaitForEncodedFrame(320, 180);
3318 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3319
3320 // Trigger adapt down, expect reduced fps (320x180@7fps).
3321 video_stream_encoder_->TriggerQualityLow();
3322 timestamp_ms += kFrameIntervalMs;
3323 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3324 sink_.WaitForEncodedFrame(timestamp_ms);
3325 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3326 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3328 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3329
3330 // Source requests VGA, expect increased resolution (640x360@7fps).
3331 source.OnOutputFormatRequest(640, 360);
3332 timestamp_ms += kFrameIntervalMs;
3333 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3334 WaitForEncodedFrame(timestamp_ms);
3335 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3336
3337 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3338 video_stream_encoder_->TriggerQualityHigh();
3339 timestamp_ms += kFrameIntervalMs;
3340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3341 WaitForEncodedFrame(timestamp_ms);
3342 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3344 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3345 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3346
3347 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3348 video_stream_encoder_->TriggerQualityHigh();
3349 timestamp_ms += kFrameIntervalMs;
3350 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3351 WaitForEncodedFrame(timestamp_ms);
3352 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3353 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3354 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3355 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3356
3357 // Trigger adapt up, expect increased fps (640x360@maxfps).
3358 video_stream_encoder_->TriggerQualityHigh();
3359 timestamp_ms += kFrameIntervalMs;
3360 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3361 WaitForEncodedFrame(timestamp_ms);
3362 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3363 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3364 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3365 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3366
3367 video_stream_encoder_->Stop();
3368}
3369
3370TEST_F(VideoStreamEncoderTest,
3371 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3372 const int kWidth = 1280;
3373 const int kHeight = 720;
3374 const int64_t kFrameIntervalMs = 150;
3375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003376 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003377
3378 // Enable BALANCED preference, no initial limitation.
3379 AdaptingFrameForwarder source(&time_controller_);
3380 source.set_adaptation_enabled(true);
3381 video_stream_encoder_->SetSource(&source,
3382 webrtc::DegradationPreference::BALANCED);
3383
3384 int64_t timestamp_ms = kFrameIntervalMs;
3385 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3386 sink_.WaitForEncodedFrame(kWidth, kHeight);
3387 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3388 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3390 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3391
3392 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3393 video_stream_encoder_->TriggerQualityLow();
3394 timestamp_ms += kFrameIntervalMs;
3395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3396 sink_.WaitForEncodedFrame(timestamp_ms);
3397 EXPECT_THAT(source.sink_wants(),
3398 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3401 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3402
3403 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3404 video_stream_encoder_->TriggerQualityLow();
3405 timestamp_ms += kFrameIntervalMs;
3406 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3407 sink_.WaitForEncodedFrame(timestamp_ms);
3408 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3410 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3411 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3412
3413 // Trigger adapt down, expect reduced fps (640x360@15fps).
3414 video_stream_encoder_->TriggerQualityLow();
3415 timestamp_ms += kFrameIntervalMs;
3416 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3417 WaitForEncodedFrame(timestamp_ms);
3418 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3420 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3421 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3424 source.OnOutputFormatRequest(320, 180);
3425 timestamp_ms += kFrameIntervalMs;
3426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3427 WaitForEncodedFrame(320, 180);
3428 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3430
3431 // Trigger adapt down, expect reduced fps (320x180@7fps).
3432 video_stream_encoder_->TriggerCpuOveruse();
3433 timestamp_ms += kFrameIntervalMs;
3434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3435 WaitForEncodedFrame(timestamp_ms);
3436 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3437 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3439 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3440 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3441 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3442 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3443
3444 // Source requests HD, expect increased resolution (640x360@7fps).
3445 source.OnOutputFormatRequest(1280, 720);
3446 timestamp_ms += kFrameIntervalMs;
3447 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3448 WaitForEncodedFrame(timestamp_ms);
3449 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3450 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3451
3452 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3453 video_stream_encoder_->TriggerCpuUnderuse();
3454 timestamp_ms += kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 WaitForEncodedFrame(timestamp_ms);
3457 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3459 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3460 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3461 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3462 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3463 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3464
3465 // Trigger adapt up, expect increased fps (640x360@maxfps).
3466 video_stream_encoder_->TriggerQualityHigh();
3467 video_stream_encoder_->TriggerCpuUnderuse();
3468 timestamp_ms += kFrameIntervalMs;
3469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3470 WaitForEncodedFrame(timestamp_ms);
3471 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3472 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3474 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3475 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3476 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3478
3479 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3480 video_stream_encoder_->TriggerQualityHigh();
3481 video_stream_encoder_->TriggerCpuUnderuse();
3482 timestamp_ms += kFrameIntervalMs;
3483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3484 WaitForEncodedFrame(timestamp_ms);
3485 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3486 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3489 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3490 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3491 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3492
3493 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3494 video_stream_encoder_->TriggerQualityHigh();
3495 video_stream_encoder_->TriggerCpuUnderuse();
3496 timestamp_ms += kFrameIntervalMs;
3497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3498 WaitForEncodedFrame(timestamp_ms);
3499 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3502 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3503 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3504 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3505 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3506
3507 video_stream_encoder_->Stop();
3508}
3509
3510TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003511 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003512 const int kWidth = 1280;
3513 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003516
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003517 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003518 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003519 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003520 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003521
3522 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003523 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003524 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003525 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3526 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3527
3528 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003529 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003530 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3532 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3533
mflodmancc3d4422017-08-03 08:27:51 -07003534 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003535}
3536
mflodmancc3d4422017-08-03 08:27:51 -07003537TEST_F(VideoStreamEncoderTest,
3538 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003539 const int kWidth = 1280;
3540 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003541 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003542 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003543
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003544 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003545 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003546 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003547 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003548
3549 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003550 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003551 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003553 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3554
3555 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003556 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003557 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003558 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003559 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3560
mflodmancc3d4422017-08-03 08:27:51 -07003561 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003562}
3563
mflodmancc3d4422017-08-03 08:27:51 -07003564TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003565 const int kWidth = 1280;
3566 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003568 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003569
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003570 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003571 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003572 video_stream_encoder_->SetSource(&source,
3573 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003574
3575 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3576 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003577 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3579 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3580 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3581
3582 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003583 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003584 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3587 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3588
mflodmancc3d4422017-08-03 08:27:51 -07003589 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003590}
3591
mflodmancc3d4422017-08-03 08:27:51 -07003592TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003593 const int kWidth = 1280;
3594 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003596 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003597
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003598 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003599 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003600 video_stream_encoder_->SetSource(&source,
3601 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003602
3603 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3604 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003605 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3608 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3609
3610 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003611 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003612 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003613 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3614 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3615 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3616
mflodmancc3d4422017-08-03 08:27:51 -07003617 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003618}
3619
mflodmancc3d4422017-08-03 08:27:51 -07003620TEST_F(VideoStreamEncoderTest,
3621 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003622 const int kWidth = 1280;
3623 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003625 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003626
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003627 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003628 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003629 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003630 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003631 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003632
3633 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003634 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003635 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003636 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3637 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3638
3639 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003640 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003641 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003642 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003643 EXPECT_THAT(source.sink_wants(),
3644 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3646 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3647
3648 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003649 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003650 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003651 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3652 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3654
mflodmancc3d4422017-08-03 08:27:51 -07003655 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003656}
3657
mflodmancc3d4422017-08-03 08:27:51 -07003658TEST_F(VideoStreamEncoderTest,
3659 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003660 const int kWidth = 1280;
3661 const int kHeight = 720;
3662 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003664 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003665
3666 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3667 stats.input_frame_rate = kInputFps;
3668 stats_proxy_->SetMockStats(stats);
3669
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003670 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003671 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3672 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003673 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003674
3675 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003676 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003677 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3678 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003679 EXPECT_THAT(video_source_.sink_wants(),
3680 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003681
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003682 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003683 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003684 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003685 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003686 // Give the encoder queue time to process the change in degradation preference
3687 // by waiting for an encoded frame.
3688 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3689 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003690 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003691
3692 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003693 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003694 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3695 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003696 EXPECT_THAT(new_video_source.sink_wants(),
3697 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003698
3699 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003700 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003701 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003702
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003704}
3705
mflodmancc3d4422017-08-03 08:27:51 -07003706TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003707 const int kWidth = 1280;
3708 const int kHeight = 720;
3709 const size_t kNumFrames = 10;
3710
Henrik Boström381d1092020-05-12 18:49:07 +02003711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003712 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003713
asaperssond0de2952017-04-21 01:47:31 -07003714 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003715 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003716 video_source_.set_adaptation_enabled(true);
3717
3718 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3720
3721 int downscales = 0;
3722 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003723 video_source_.IncomingCapturedFrame(
3724 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3725 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003726
asaperssonfab67072017-04-04 05:51:49 -07003727 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003728 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003729 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003730 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003731
3732 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3733 ++downscales;
3734
3735 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3736 EXPECT_EQ(downscales,
3737 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3738 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003739 }
mflodmancc3d4422017-08-03 08:27:51 -07003740 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003741}
3742
mflodmancc3d4422017-08-03 08:27:51 -07003743TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003744 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3745 const int kWidth = 1280;
3746 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003747 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003748 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003749
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003750 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003751 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003752 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003753 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003754 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003755
Åsa Persson8c1bf952018-09-13 10:42:19 +02003756 int64_t timestamp_ms = kFrameIntervalMs;
3757 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003758 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003760 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3761 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3762
3763 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003764 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003765 timestamp_ms += kFrameIntervalMs;
3766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3767 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003768 EXPECT_THAT(source.sink_wants(),
3769 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003770 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3771 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3772
3773 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003774 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003775 timestamp_ms += kFrameIntervalMs;
3776 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003777 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003779 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3780 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3781
3782 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003783 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003784 timestamp_ms += kFrameIntervalMs;
3785 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3786 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003787 EXPECT_THAT(source.sink_wants(),
3788 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003789 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3790 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3791
3792 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003793 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003794 timestamp_ms += kFrameIntervalMs;
3795 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003796 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003797 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3799 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3800
mflodmancc3d4422017-08-03 08:27:51 -07003801 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003802}
3803
mflodmancc3d4422017-08-03 08:27:51 -07003804TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003805 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3806 const int kWidth = 1280;
3807 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003810
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003811 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003812 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003813 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003814 video_stream_encoder_->SetSource(&source,
3815 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003816
Åsa Persson8c1bf952018-09-13 10:42:19 +02003817 int64_t timestamp_ms = kFrameIntervalMs;
3818 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003819 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003820 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3822 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823
3824 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003825 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003826 timestamp_ms += kFrameIntervalMs;
3827 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3828 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003829 EXPECT_THAT(source.sink_wants(),
3830 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003831 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3832 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3833
3834 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003835 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003836 timestamp_ms += kFrameIntervalMs;
3837 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003838 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003839 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3841 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3842
3843 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003844 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003845 timestamp_ms += kFrameIntervalMs;
3846 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3847 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003848 EXPECT_THAT(source.sink_wants(),
3849 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003850 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3851 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3852
3853 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003854 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003855 timestamp_ms += kFrameIntervalMs;
3856 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003857 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003858 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003859 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3860 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3861
mflodmancc3d4422017-08-03 08:27:51 -07003862 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003863}
3864
Sergey Silkin41c650b2019-10-14 13:12:19 +02003865TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3866 fake_encoder_.SetResolutionBitrateLimits(
3867 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3868
Henrik Boström381d1092020-05-12 18:49:07 +02003869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003870 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3871 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3872 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3873 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003874
3875 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003876 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003877 source.set_adaptation_enabled(true);
3878 video_stream_encoder_->SetSource(
3879 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3880
3881 // Insert 720p frame.
3882 int64_t timestamp_ms = kFrameIntervalMs;
3883 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3884 WaitForEncodedFrame(1280, 720);
3885
3886 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003888 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3889 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3890 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3891 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003892 video_stream_encoder_->TriggerQualityLow();
3893
3894 // Insert 720p frame. It should be downscaled and encoded.
3895 timestamp_ms += kFrameIntervalMs;
3896 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3897 WaitForEncodedFrame(960, 540);
3898
3899 // Trigger adapt up. Higher resolution should not be requested duo to lack
3900 // of bitrate.
3901 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003902 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003903
3904 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003905 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003906 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3907 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3908 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3909 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003910
3911 // Trigger adapt up. Higher resolution should be requested.
3912 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003913 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003914
3915 video_stream_encoder_->Stop();
3916}
3917
3918TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3919 fake_encoder_.SetResolutionBitrateLimits(
3920 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3921
3922 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003924 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3925 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3926 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3927 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003928
3929 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003930 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003931 source.set_adaptation_enabled(true);
3932 video_stream_encoder_->SetSource(
3933 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3934
3935 // Insert 720p frame. It should be dropped and lower resolution should be
3936 // requested.
3937 int64_t timestamp_ms = kFrameIntervalMs;
3938 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3939 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003940 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003941
3942 // Insert 720p frame. It should be downscaled and encoded.
3943 timestamp_ms += kFrameIntervalMs;
3944 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3945 WaitForEncodedFrame(960, 540);
3946
3947 video_stream_encoder_->Stop();
3948}
3949
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003950class BalancedDegradationTest : public VideoStreamEncoderTest {
3951 protected:
3952 void SetupTest() {
3953 // Reset encoder for field trials to take effect.
3954 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02003955 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003956
3957 // Enable BALANCED preference.
3958 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003959 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3960 }
3961
Asa Persson606d3cb2021-10-04 10:07:11 +02003962 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02003963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003964 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003965 }
3966
Åsa Persson45b176f2019-09-30 11:19:05 +02003967 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003968 timestamp_ms_ += kFrameIntervalMs;
3969 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003970 }
3971
3972 void InsertFrameAndWaitForEncoded() {
3973 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003974 sink_.WaitForEncodedFrame(timestamp_ms_);
3975 }
3976
3977 const int kWidth = 640; // pixels:640x360=230400
3978 const int kHeight = 360;
3979 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3980 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003981 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003982};
3983
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003984TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003985 test::ScopedFieldTrials field_trials(
3986 "WebRTC-Video-BalancedDegradationSettings/"
3987 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3988 SetupTest();
3989
3990 // Force input frame rate.
3991 const int kInputFps = 24;
3992 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3993 stats.input_frame_rate = kInputFps;
3994 stats_proxy_->SetMockStats(stats);
3995
Åsa Persson45b176f2019-09-30 11:19:05 +02003996 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003997 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003998
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003999 // Trigger adapt down, expect scaled down framerate and resolution,
4000 // since Fps diff (input-requested:0) < threshold.
4001 video_stream_encoder_->TriggerQualityLow();
4002 EXPECT_THAT(source_.sink_wants(),
4003 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004004
4005 video_stream_encoder_->Stop();
4006}
4007
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004008TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004009 test::ScopedFieldTrials field_trials(
4010 "WebRTC-Video-BalancedDegradationSettings/"
4011 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4012 SetupTest();
4013
4014 // Force input frame rate.
4015 const int kInputFps = 25;
4016 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4017 stats.input_frame_rate = kInputFps;
4018 stats_proxy_->SetMockStats(stats);
4019
Åsa Persson45b176f2019-09-30 11:19:05 +02004020 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004021 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004022
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004023 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4024 // Fps diff (input-requested:1) == threshold.
4025 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004026 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004027
4028 video_stream_encoder_->Stop();
4029}
4030
4031TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4032 test::ScopedFieldTrials field_trials(
4033 "WebRTC-Video-BalancedDegradationSettings/"
4034 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4035 SetupTest();
4036
4037 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4038
Åsa Persson45b176f2019-09-30 11:19:05 +02004039 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004040 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004041
4042 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4043 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004044 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004045
4046 video_stream_encoder_->Stop();
4047}
4048
Åsa Perssonccfb3402019-09-25 15:13:04 +02004049TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004050 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004051 "WebRTC-Video-BalancedDegradationSettings/"
4052 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004053 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004054
Asa Persson606d3cb2021-10-04 10:07:11 +02004055 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4056 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4057 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004058
Åsa Persson45b176f2019-09-30 11:19:05 +02004059 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004060 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004061 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4062
4063 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4064 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004065 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004066 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004067 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4068
4069 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4070 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004071 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004072 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004073 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4074
Åsa Persson30ab0152019-08-27 12:22:33 +02004075 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4076 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004077 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004078 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004079 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004080 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4081
4082 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004083 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004084 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004085 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004086
Åsa Persson30ab0152019-08-27 12:22:33 +02004087 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004088 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004089 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004090 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004091 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004092 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4093
4094 video_stream_encoder_->Stop();
4095}
4096
Åsa Perssonccfb3402019-09-25 15:13:04 +02004097TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004098 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4099 test::ScopedFieldTrials field_trials(
4100 "WebRTC-Video-BalancedDegradationSettings/"
4101 "pixels:57600|129600|230400,fps:7|24|24/");
4102 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004103 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004104
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004105 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004106
4107 // Insert frame, expect scaled down:
4108 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4109 InsertFrame();
4110 EXPECT_FALSE(WaitForFrame(1000));
4111 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4112 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4113
4114 // Insert frame, expect scaled down:
4115 // resolution (320x180@24fps).
4116 InsertFrame();
4117 EXPECT_FALSE(WaitForFrame(1000));
4118 EXPECT_LT(source_.sink_wants().max_pixel_count,
4119 source_.last_wants().max_pixel_count);
4120 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4121
4122 // Frame should not be dropped (min pixels per frame reached).
4123 InsertFrameAndWaitForEncoded();
4124
4125 video_stream_encoder_->Stop();
4126}
4127
4128TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004129 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004130 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004131 "WebRTC-Video-BalancedDegradationSettings/"
4132 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004133 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004134
Asa Persson606d3cb2021-10-04 10:07:11 +02004135 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4136 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4137 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004138
Åsa Persson45b176f2019-09-30 11:19:05 +02004139 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004140 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004141 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4142
4143 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4144 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004145 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004146 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004147 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4148
4149 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4150 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004151 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004152 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004153 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4154
4155 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4156 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004157 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004158 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004159 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4160
Åsa Persson30ab0152019-08-27 12:22:33 +02004161 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4162 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004163 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004164 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004165 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4166
4167 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4168 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004169 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004170 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4171
4172 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004173 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004174 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004175 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004176 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004177 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4178
4179 video_stream_encoder_->Stop();
4180}
4181
Åsa Perssonccfb3402019-09-25 15:13:04 +02004182TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004183 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004184 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004185 "WebRTC-Video-BalancedDegradationSettings/"
4186 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004187 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004188
Asa Persson606d3cb2021-10-04 10:07:11 +02004189 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4190 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4191 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4192 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4193 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004194
Åsa Persson45b176f2019-09-30 11:19:05 +02004195 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004196 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004197 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4198
4199 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4200 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004201 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004202 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004203 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4204
4205 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4206 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004207 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004208 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004209 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4210
4211 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4212 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004213 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004214 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004215 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4216
4217 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4218 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004219 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004220 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4221
4222 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004223 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004224 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004225 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004226 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004227 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4228
4229 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004230 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004231 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004232 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004233 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4234
4235 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004236 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004237 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004238 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004239 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4241
Åsa Persson1b247f12019-08-14 17:26:39 +02004242 video_stream_encoder_->Stop();
4243}
4244
mflodmancc3d4422017-08-03 08:27:51 -07004245TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004246 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4247 const int kWidth = 1280;
4248 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004249 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004250 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004251
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004252 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004253 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004254 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004256 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004257
Åsa Persson8c1bf952018-09-13 10:42:19 +02004258 int64_t timestamp_ms = kFrameIntervalMs;
4259 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004260 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004261 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004262 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4263 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4264 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4265 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4266
4267 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004268 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004269 timestamp_ms += kFrameIntervalMs;
4270 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4271 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004272 EXPECT_THAT(source.sink_wants(),
4273 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004274 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4275 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4276 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4277 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4278
4279 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004280 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004281 timestamp_ms += kFrameIntervalMs;
4282 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4283 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004284 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004285 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4287 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4288 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4289
Jonathan Yubc771b72017-12-08 17:04:29 -08004290 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004291 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004292 timestamp_ms += kFrameIntervalMs;
4293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4294 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004295 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004296 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004298 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004299 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4300
Jonathan Yubc771b72017-12-08 17:04:29 -08004301 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004302 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004303 timestamp_ms += kFrameIntervalMs;
4304 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4305 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004306 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004307 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004308 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4309 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4310 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4311 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4312
Jonathan Yubc771b72017-12-08 17:04:29 -08004313 // Trigger quality adapt down, expect no change (min resolution reached).
4314 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004315 timestamp_ms += kFrameIntervalMs;
4316 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4317 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004318 EXPECT_THAT(source.sink_wants(), FpsMax());
4319 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004320 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4321 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4322 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4323 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4324
Evan Shrubsole64469032020-06-11 10:45:29 +02004325 // Trigger quality adapt up, expect upscaled resolution (480x270).
4326 video_stream_encoder_->TriggerQualityHigh();
4327 timestamp_ms += kFrameIntervalMs;
4328 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4329 WaitForEncodedFrame(timestamp_ms);
4330 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4331 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4333 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4334 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4335
4336 // Trigger quality and cpu adapt up since both are most limited, expect
4337 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004338 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004340 timestamp_ms += kFrameIntervalMs;
4341 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4342 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004343 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004344 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4345 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4346 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004347 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004348
Evan Shrubsole64469032020-06-11 10:45:29 +02004349 // Trigger quality and cpu adapt up since both are most limited, expect
4350 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004351 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004352 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004353 timestamp_ms += kFrameIntervalMs;
4354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004356 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004357 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004358 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004359 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004360 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4361 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004362
Evan Shrubsole64469032020-06-11 10:45:29 +02004363 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4364 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004365 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004366 timestamp_ms += kFrameIntervalMs;
4367 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4368 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004369 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004370 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004372 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004373 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004374
4375 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004376 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004377 timestamp_ms += kFrameIntervalMs;
4378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004379 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004380 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004381 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004382 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4383 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004384 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004385 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004386
mflodmancc3d4422017-08-03 08:27:51 -07004387 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004388}
4389
mflodmancc3d4422017-08-03 08:27:51 -07004390TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004391 const int kWidth = 640;
4392 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004393
Henrik Boström381d1092020-05-12 18:49:07 +02004394 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004395 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004396
perkj803d97f2016-11-01 11:45:46 -07004397 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004398 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004399 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004400 }
4401
mflodmancc3d4422017-08-03 08:27:51 -07004402 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004403 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004404 video_source_.IncomingCapturedFrame(CreateFrame(
4405 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004406 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004407 }
4408
mflodmancc3d4422017-08-03 08:27:51 -07004409 video_stream_encoder_->Stop();
4410 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004411 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004412
Ying Wangef3998f2019-12-09 13:06:53 +01004413 EXPECT_METRIC_EQ(
4414 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4415 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004416 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4417}
4418
mflodmancc3d4422017-08-03 08:27:51 -07004419TEST_F(VideoStreamEncoderTest,
4420 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004422 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004423 const int kWidth = 640;
4424 const int kHeight = 360;
4425
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004426 video_stream_encoder_->SetSource(&video_source_,
4427 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004428
4429 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4430 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004431 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004432 }
4433
mflodmancc3d4422017-08-03 08:27:51 -07004434 video_stream_encoder_->Stop();
4435 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004436 stats_proxy_.reset();
4437
4438 EXPECT_EQ(0,
4439 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4440}
4441
Per Kjellanderdcef6412020-10-07 15:09:05 +02004442TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4443 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004444 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004445 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004446
4447 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004448 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004449 SimulcastRateAllocator(fake_encoder_.config())
4450 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004451 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004452
Henrik Boström381d1092020-05-12 18:49:07 +02004453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004454 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004455
sprang57c2fff2017-01-16 06:24:02 -08004456 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004457 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4458 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004459 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4460 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4461
Erik Språngd7329ca2019-02-21 21:19:53 +01004462 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004463 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004464 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004465
Per Kjellanderdcef6412020-10-07 15:09:05 +02004466 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004467 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004468 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4469 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004470 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004471 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004472
Per Kjellanderdcef6412020-10-07 15:09:05 +02004473 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004474 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004475 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004476 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004477 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4478 WaitForEncodedFrame(CurrentTimeMs());
4479 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004480 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004481 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004482
mflodmancc3d4422017-08-03 08:27:51 -07004483 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004484}
4485
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004486TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004487 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004488 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004489 kVideoLayersAllocation);
4490
4491 const int kDefaultFps = 30;
4492
4493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004494 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004495
4496 video_source_.IncomingCapturedFrame(
4497 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4498 WaitForEncodedFrame(CurrentTimeMs());
4499 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4500 VideoLayersAllocation last_layer_allocation =
4501 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004502 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004503 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4504
4505 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004506 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004507 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004508 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004509 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4510
Erik Språng9d69cbe2020-10-22 17:44:42 +02004511 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004512 int number_of_layers_allocation = 1;
4513 const int64_t start_time_ms = CurrentTimeMs();
4514 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4515 video_source_.IncomingCapturedFrame(
4516 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4517 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004518 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4519 number_of_layers_allocation = sink_.number_of_layers_allocations();
4520 VideoLayersAllocation new_allocation =
4521 sink_.GetLastVideoLayersAllocation();
4522 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4523 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4524 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4525 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4526 .target_bitrate_per_temporal_layer,
4527 last_layer_allocation.active_spatial_layers[0]
4528 .target_bitrate_per_temporal_layer);
4529 last_layer_allocation = new_allocation;
4530 }
4531 }
4532 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4533 video_stream_encoder_->Stop();
4534}
4535
4536TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004537 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004538 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4539 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4540 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004541 VideoEncoderConfig video_encoder_config;
4542 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4543 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004544 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004545 video_encoder_config.content_type =
4546 VideoEncoderConfig::ContentType::kRealtimeVideo;
4547 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004548 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004549 VideoEncoder::GetDefaultVp8Settings());
4550 for (auto& layer : video_encoder_config.simulcast_layers) {
4551 layer.num_temporal_layers = 2;
4552 }
4553 // Simulcast layers are used for enabling/disabling streams.
4554 video_encoder_config.simulcast_layers[0].active = true;
4555 video_encoder_config.simulcast_layers[1].active = false;
4556 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004557 ConfigureEncoder(std::move(video_encoder_config),
4558 VideoStreamEncoder::BitrateAllocationCallbackType::
4559 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004560
4561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004562 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004563
4564 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4565 WaitForEncodedFrame(CurrentTimeMs());
4566 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4567 VideoLayersAllocation last_layer_allocation =
4568 sink_.GetLastVideoLayersAllocation();
4569
4570 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4571 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4572 .target_bitrate_per_temporal_layer,
4573 SizeIs(2));
4574 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4575 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4576 video_stream_encoder_->Stop();
4577}
4578
4579TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004580 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004581 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4582 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4583 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004584 VideoEncoderConfig video_encoder_config;
4585 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4586 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004587 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004588 video_encoder_config.content_type =
4589 VideoEncoderConfig::ContentType::kRealtimeVideo;
4590 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004591 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004592 VideoEncoder::GetDefaultVp8Settings());
4593 for (auto& layer : video_encoder_config.simulcast_layers) {
4594 layer.num_temporal_layers = 2;
4595 }
4596 // Simulcast layers are used for enabling/disabling streams.
4597 video_encoder_config.simulcast_layers[0].active = true;
4598 video_encoder_config.simulcast_layers[1].active = false;
4599 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004600 ConfigureEncoder(std::move(video_encoder_config),
4601 VideoStreamEncoder::BitrateAllocationCallbackType::
4602 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004603
4604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004605 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004606
4607 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4608 WaitForEncodedFrame(CurrentTimeMs());
4609 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4610 VideoLayersAllocation last_layer_allocation =
4611 sink_.GetLastVideoLayersAllocation();
4612
4613 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4614 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4615 .target_bitrate_per_temporal_layer,
4616 SizeIs(2));
4617 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4618
4619 video_stream_encoder_->Stop();
4620}
4621
4622TEST_F(VideoStreamEncoderTest,
4623 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4624 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4625 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004626 VideoEncoderConfig video_encoder_config;
4627 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4628 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004629 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004630 video_encoder_config.content_type =
4631 VideoEncoderConfig::ContentType::kRealtimeVideo;
4632 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4633 vp9_settings.numberOfSpatialLayers = 2;
4634 vp9_settings.numberOfTemporalLayers = 2;
4635 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4636 vp9_settings.automaticResizeOn = false;
4637 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004638 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004639 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004640 ConfigureEncoder(std::move(video_encoder_config),
4641 VideoStreamEncoder::BitrateAllocationCallbackType::
4642 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004643
4644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004646
4647 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4648 WaitForEncodedFrame(CurrentTimeMs());
4649 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4650 VideoLayersAllocation last_layer_allocation =
4651 sink_.GetLastVideoLayersAllocation();
4652
4653 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4654 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4655 .target_bitrate_per_temporal_layer,
4656 SizeIs(2));
4657 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4658 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4659 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4660 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4661 .target_bitrate_per_temporal_layer,
4662 SizeIs(2));
4663 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4664 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4665 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4666
4667 // Since full SVC is used, expect the top layer to utilize the full target
4668 // rate.
4669 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4670 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004671 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004672 video_stream_encoder_->Stop();
4673}
4674
4675TEST_F(VideoStreamEncoderTest,
4676 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4677 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4678 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004679 VideoEncoderConfig video_encoder_config;
4680 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4681 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004682 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004683 video_encoder_config.content_type =
4684 VideoEncoderConfig::ContentType::kRealtimeVideo;
4685 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4686 vp9_settings.numberOfSpatialLayers = 2;
4687 vp9_settings.numberOfTemporalLayers = 2;
4688 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4689 vp9_settings.automaticResizeOn = false;
4690 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004691 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004692 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004693 ConfigureEncoder(std::move(video_encoder_config),
4694 VideoStreamEncoder::BitrateAllocationCallbackType::
4695 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004696
4697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004698 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004699
4700 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4701 WaitForEncodedFrame(CurrentTimeMs());
4702 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4703 VideoLayersAllocation last_layer_allocation =
4704 sink_.GetLastVideoLayersAllocation();
4705
4706 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4707 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4708 .target_bitrate_per_temporal_layer,
4709 SizeIs(1));
4710 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4711 .target_bitrate_per_temporal_layer,
4712 SizeIs(1));
4713 // Since full SVC is used, expect the top layer to utilize the full target
4714 // rate.
4715 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4716 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004717 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 video_stream_encoder_->Stop();
4719}
4720
4721TEST_F(VideoStreamEncoderTest,
4722 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4723 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4724 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725 VideoEncoderConfig video_encoder_config;
4726 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4727 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004728 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 video_encoder_config.content_type =
4730 VideoEncoderConfig::ContentType::kRealtimeVideo;
4731 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4732 vp9_settings.numberOfSpatialLayers = 2;
4733 vp9_settings.numberOfTemporalLayers = 2;
4734 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4735 vp9_settings.automaticResizeOn = false;
4736 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004737 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004738 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004739 ConfigureEncoder(std::move(video_encoder_config),
4740 VideoStreamEncoder::BitrateAllocationCallbackType::
4741 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742
4743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004745
4746 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4747 WaitForEncodedFrame(CurrentTimeMs());
4748 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4749 VideoLayersAllocation last_layer_allocation =
4750 sink_.GetLastVideoLayersAllocation();
4751
4752 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4753 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4754 .target_bitrate_per_temporal_layer,
4755 SizeIs(2));
4756 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4757 .target_bitrate_per_temporal_layer,
4758 SizeIs(2));
4759 // Since KSVC is, spatial layers are independend except on key frames.
4760 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4761 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004762 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004763 video_stream_encoder_->Stop();
4764}
4765
4766TEST_F(VideoStreamEncoderTest,
4767 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4768 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4769 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4770 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004771 VideoEncoderConfig video_encoder_config;
4772 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4773 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004774 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775 video_encoder_config.content_type =
4776 VideoEncoderConfig::ContentType::kRealtimeVideo;
4777 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4778 vp9_settings.numberOfSpatialLayers = 3;
4779 vp9_settings.numberOfTemporalLayers = 2;
4780 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4781 vp9_settings.automaticResizeOn = false;
4782 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004783 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004784 vp9_settings);
4785 // Simulcast layers are used for enabling/disabling streams.
4786 video_encoder_config.simulcast_layers.resize(3);
4787 video_encoder_config.simulcast_layers[0].active = false;
4788 video_encoder_config.simulcast_layers[1].active = true;
4789 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004790 ConfigureEncoder(std::move(video_encoder_config),
4791 VideoStreamEncoder::BitrateAllocationCallbackType::
4792 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004793
4794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004795 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004796
4797 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4798 WaitForEncodedFrame(CurrentTimeMs());
4799 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4800 VideoLayersAllocation last_layer_allocation =
4801 sink_.GetLastVideoLayersAllocation();
4802
4803 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4804 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4805 .target_bitrate_per_temporal_layer,
4806 SizeIs(2));
4807 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4808 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4809
4810 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4811 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4812 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4813 .target_bitrate_per_temporal_layer,
4814 SizeIs(2));
4815 // Since full SVC is used, expect the top layer to utilize the full target
4816 // rate.
4817 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4818 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004819 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004820 video_stream_encoder_->Stop();
4821}
4822
4823TEST_F(VideoStreamEncoderTest,
4824 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4825 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4826 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4827 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004828 VideoEncoderConfig video_encoder_config;
4829 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4830 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004831 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004832 video_encoder_config.content_type =
4833 VideoEncoderConfig::ContentType::kRealtimeVideo;
4834 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4835 vp9_settings.numberOfSpatialLayers = 3;
4836 vp9_settings.numberOfTemporalLayers = 2;
4837 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4838 vp9_settings.automaticResizeOn = false;
4839 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004840 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004841 vp9_settings);
4842 // Simulcast layers are used for enabling/disabling streams.
4843 video_encoder_config.simulcast_layers.resize(3);
4844 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004845 ConfigureEncoder(std::move(video_encoder_config),
4846 VideoStreamEncoder::BitrateAllocationCallbackType::
4847 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004848
4849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004850 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851
4852 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4853 WaitForEncodedFrame(CurrentTimeMs());
4854 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4855 VideoLayersAllocation last_layer_allocation =
4856 sink_.GetLastVideoLayersAllocation();
4857
4858 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4859 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4860 .target_bitrate_per_temporal_layer,
4861 SizeIs(2));
4862 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4863 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4864
4865 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4866 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4867 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4868 .target_bitrate_per_temporal_layer,
4869 SizeIs(2));
4870 video_stream_encoder_->Stop();
4871}
4872
4873TEST_F(VideoStreamEncoderTest,
4874 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4875 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4876 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4877 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004878 VideoEncoderConfig video_encoder_config;
4879 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4880 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004881 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004882 video_encoder_config.content_type =
4883 VideoEncoderConfig::ContentType::kRealtimeVideo;
4884 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4885 vp9_settings.numberOfSpatialLayers = 3;
4886 vp9_settings.numberOfTemporalLayers = 2;
4887 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4888 vp9_settings.automaticResizeOn = false;
4889 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004890 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004891 vp9_settings);
4892 // Simulcast layers are used for enabling/disabling streams.
4893 video_encoder_config.simulcast_layers.resize(3);
4894 video_encoder_config.simulcast_layers[0].active = false;
4895 video_encoder_config.simulcast_layers[1].active = false;
4896 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004897 ConfigureEncoder(std::move(video_encoder_config),
4898 VideoStreamEncoder::BitrateAllocationCallbackType::
4899 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004900
4901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004902 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004903
4904 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4905 WaitForEncodedFrame(CurrentTimeMs());
4906 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4907 VideoLayersAllocation last_layer_allocation =
4908 sink_.GetLastVideoLayersAllocation();
4909
4910 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4911 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4912 .target_bitrate_per_temporal_layer,
4913 SizeIs(2));
4914 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4915 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4916 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4917 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004918 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004919 video_stream_encoder_->Stop();
4920}
4921
4922TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4923 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004924 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004925 kVideoLayersAllocation);
4926 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004927 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004928
4929 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4930 WaitForEncodedFrame(CurrentTimeMs());
4931 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4932 VideoLayersAllocation last_layer_allocation =
4933 sink_.GetLastVideoLayersAllocation();
4934
4935 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4936 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4937 .target_bitrate_per_temporal_layer,
4938 SizeIs(1));
4939 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4940 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004941 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004942 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4943 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4945 video_stream_encoder_->Stop();
4946}
4947
4948TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004949 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4950 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004951 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004952 kVideoLayersAllocation);
4953
4954 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004955 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004956
4957 video_source_.IncomingCapturedFrame(
4958 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4959 WaitForEncodedFrame(CurrentTimeMs());
4960 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4961 VideoLayersAllocation last_layer_allocation =
4962 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004963 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004964 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4965 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4966 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004967 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02004968
4969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004970 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
4971 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004972 video_source_.IncomingCapturedFrame(
4973 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4974 WaitForEncodedFrame(CurrentTimeMs());
4975
4976 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4977 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4978 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4979 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4980 .target_bitrate_per_temporal_layer[0],
4981 DataRate::Zero());
4982
4983 video_stream_encoder_->Stop();
4984}
4985
Per Kjellander4190ce92020-12-15 17:24:55 +01004986TEST_F(VideoStreamEncoderTest,
4987 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4988 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004989 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004990 kVideoLayersAllocation);
4991
4992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004993 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
4994 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01004995
4996 video_source_.IncomingCapturedFrame(
4997 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4998 WaitForEncodedFrame(CurrentTimeMs());
4999 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5000 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5001 SizeIs(2));
5002 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5003 codec_width_);
5004 EXPECT_EQ(
5005 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5006 codec_height_);
5007
5008 video_source_.IncomingCapturedFrame(
5009 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5010 WaitForEncodedFrame(CurrentTimeMs());
5011 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5012 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5013 SizeIs(2));
5014 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5015 codec_width_ / 2);
5016 EXPECT_EQ(
5017 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5018 codec_height_ / 2);
5019
5020 video_stream_encoder_->Stop();
5021}
5022
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005023TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5024 // 2 TLs configured, temporal layers supported by encoder.
5025 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005026 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005027 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005028 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005029 fake_encoder_.SetTemporalLayersSupported(0, true);
5030
5031 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005032 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005033 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005034 kNumTemporalLayers, /*temporal_id*/ 0,
5035 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005036 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005037 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005038 kNumTemporalLayers, /*temporal_id*/ 1,
5039 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005040 VideoBitrateAllocation expected_bitrate;
5041 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5042 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5043
5044 VerifyAllocatedBitrate(expected_bitrate);
5045 video_stream_encoder_->Stop();
5046}
5047
5048TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5049 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005050 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005051 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005052 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005053 fake_encoder_.SetTemporalLayersSupported(0, false);
5054
5055 // Temporal layers not supported by the encoder.
5056 // Total bitrate should be at ti:0.
5057 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005058 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005059
5060 VerifyAllocatedBitrate(expected_bitrate);
5061 video_stream_encoder_->Stop();
5062}
5063
5064TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005065 webrtc::test::ScopedFieldTrials field_trials(
5066 "WebRTC-Video-QualityScalerSettings/"
5067 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5068 // Reset encoder for field trials to take effect.
5069 ConfigureEncoder(video_encoder_config_.Copy());
5070
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005071 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005072 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005073 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005074 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005075 fake_encoder_.SetTemporalLayersSupported(0, true);
5076 fake_encoder_.SetTemporalLayersSupported(1, false);
5077
5078 const int kS0Bps = 150000;
5079 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005080 kS0Bps *
5081 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5082 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005083 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005084 kS0Bps *
5085 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5086 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005087 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005088 // Temporal layers not supported by si:1.
5089 VideoBitrateAllocation expected_bitrate;
5090 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5091 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5092 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5093
5094 VerifyAllocatedBitrate(expected_bitrate);
5095 video_stream_encoder_->Stop();
5096}
5097
Niels Möller7dc26b72017-12-06 10:27:48 +01005098TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5099 const int kFrameWidth = 1280;
5100 const int kFrameHeight = 720;
5101 const int kFramerate = 24;
5102
Henrik Boström381d1092020-05-12 18:49:07 +02005103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005104 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005105 test::FrameForwarder source;
5106 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005107 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005108
5109 // Insert a single frame, triggering initial configuration.
5110 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5111 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5112
5113 EXPECT_EQ(
5114 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5115 kDefaultFramerate);
5116
5117 // Trigger reconfigure encoder (without resetting the entire instance).
5118 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005119 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5120 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005121 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005122 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005123 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005124 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5125
5126 // Detector should be updated with fps limit from codec config.
5127 EXPECT_EQ(
5128 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5129 kFramerate);
5130
5131 // Trigger overuse, max framerate should be reduced.
5132 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5133 stats.input_frame_rate = kFramerate;
5134 stats_proxy_->SetMockStats(stats);
5135 video_stream_encoder_->TriggerCpuOveruse();
5136 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5137 int adapted_framerate =
5138 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5139 EXPECT_LT(adapted_framerate, kFramerate);
5140
5141 // Trigger underuse, max framerate should go back to codec configured fps.
5142 // Set extra low fps, to make sure it's actually reset, not just incremented.
5143 stats = stats_proxy_->GetStats();
5144 stats.input_frame_rate = adapted_framerate / 2;
5145 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005146 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005147 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5148 EXPECT_EQ(
5149 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5150 kFramerate);
5151
5152 video_stream_encoder_->Stop();
5153}
5154
5155TEST_F(VideoStreamEncoderTest,
5156 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5157 const int kFrameWidth = 1280;
5158 const int kFrameHeight = 720;
5159 const int kLowFramerate = 15;
5160 const int kHighFramerate = 25;
5161
Henrik Boström381d1092020-05-12 18:49:07 +02005162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005163 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005164 test::FrameForwarder source;
5165 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005166 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005167
5168 // Trigger initial configuration.
5169 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005170 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5171 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005172 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005173 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005174 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005175 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005176 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5177
5178 EXPECT_EQ(
5179 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5180 kLowFramerate);
5181
5182 // Trigger overuse, max framerate should be reduced.
5183 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5184 stats.input_frame_rate = kLowFramerate;
5185 stats_proxy_->SetMockStats(stats);
5186 video_stream_encoder_->TriggerCpuOveruse();
5187 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5188 int adapted_framerate =
5189 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5190 EXPECT_LT(adapted_framerate, kLowFramerate);
5191
5192 // Reconfigure the encoder with a new (higher max framerate), max fps should
5193 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005194 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005195 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5196 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005197 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005198 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5199
5200 EXPECT_EQ(
5201 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5202 adapted_framerate);
5203
5204 // Trigger underuse, max framerate should go back to codec configured fps.
5205 stats = stats_proxy_->GetStats();
5206 stats.input_frame_rate = adapted_framerate;
5207 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005208 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005209 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5210 EXPECT_EQ(
5211 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5212 kHighFramerate);
5213
5214 video_stream_encoder_->Stop();
5215}
5216
mflodmancc3d4422017-08-03 08:27:51 -07005217TEST_F(VideoStreamEncoderTest,
5218 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005219 const int kFrameWidth = 1280;
5220 const int kFrameHeight = 720;
5221 const int kFramerate = 24;
5222
Henrik Boström381d1092020-05-12 18:49:07 +02005223 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005224 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005225 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005226 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005227 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005228
5229 // Trigger initial configuration.
5230 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005231 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5232 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005233 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005234 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005235 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005236 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005237 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005238
Niels Möller7dc26b72017-12-06 10:27:48 +01005239 EXPECT_EQ(
5240 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5241 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005242
5243 // Trigger overuse, max framerate should be reduced.
5244 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5245 stats.input_frame_rate = kFramerate;
5246 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005247 video_stream_encoder_->TriggerCpuOveruse();
5248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005249 int adapted_framerate =
5250 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005251 EXPECT_LT(adapted_framerate, kFramerate);
5252
5253 // Change degradation preference to not enable framerate scaling. Target
5254 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005255 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005256 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005257 EXPECT_EQ(
5258 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5259 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005260
mflodmancc3d4422017-08-03 08:27:51 -07005261 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005262}
5263
mflodmancc3d4422017-08-03 08:27:51 -07005264TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005265 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005267 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5268 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5269 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005270 const int kWidth = 640;
5271 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005272
asaperssonfab67072017-04-04 05:51:49 -07005273 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005274
5275 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005276 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005277
5278 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005279 EXPECT_TRUE_WAIT(
5280 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005281
sprangc5d62e22017-04-02 23:53:04 -07005282 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005283
asaperssonfab67072017-04-04 05:51:49 -07005284 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005285 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005286 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005287
5288 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005289 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005290
Henrik Boström2671dac2020-05-19 16:29:09 +02005291 EXPECT_TRUE_WAIT(
5292 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005293
mflodmancc3d4422017-08-03 08:27:51 -07005294 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005295}
5296
mflodmancc3d4422017-08-03 08:27:51 -07005297TEST_F(VideoStreamEncoderTest,
5298 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005299 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005300 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005301 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5302 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5303 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005304 const int kWidth = 640;
5305 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005306
5307 // We expect the n initial frames to get dropped.
5308 int i;
5309 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005310 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005311 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005312 }
5313 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005314 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005315 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005316
5317 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005318 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005319
mflodmancc3d4422017-08-03 08:27:51 -07005320 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005321}
5322
mflodmancc3d4422017-08-03 08:27:51 -07005323TEST_F(VideoStreamEncoderTest,
5324 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005325 const int kWidth = 640;
5326 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005327 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005328 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005329
5330 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005331 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005332 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005333
asaperssonfab67072017-04-04 05:51:49 -07005334 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005335 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005336 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005337
mflodmancc3d4422017-08-03 08:27:51 -07005338 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005339}
5340
mflodmancc3d4422017-08-03 08:27:51 -07005341TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005342 const int kWidth = 640;
5343 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005344 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005345
5346 VideoEncoderConfig video_encoder_config;
5347 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5348 // Make format different, to force recreation of encoder.
5349 video_encoder_config.video_format.parameters["foo"] = "foo";
5350 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005351 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005353 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005354
kthelgasonb83797b2017-02-14 11:57:25 -08005355 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005356 video_stream_encoder_->SetSource(&video_source_,
5357 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005358
asaperssonfab67072017-04-04 05:51:49 -07005359 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005360 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005361 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005362
mflodmancc3d4422017-08-03 08:27:51 -07005363 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005364 fake_encoder_.SetQualityScaling(true);
5365}
5366
Åsa Persson139f4dc2019-08-02 09:29:58 +02005367TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5368 webrtc::test::ScopedFieldTrials field_trials(
5369 "WebRTC-Video-QualityScalerSettings/"
5370 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5371 // Reset encoder for field trials to take effect.
5372 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005373 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5374 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005375 const int kWidth = 640;
5376 const int kHeight = 360;
5377
Henrik Boström381d1092020-05-12 18:49:07 +02005378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005379 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005380 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5381 // Frame should not be dropped.
5382 WaitForEncodedFrame(1);
5383
Henrik Boström381d1092020-05-12 18:49:07 +02005384 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005385 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5386 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5387 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005388 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5389 // Frame should not be dropped.
5390 WaitForEncodedFrame(2);
5391
Henrik Boström381d1092020-05-12 18:49:07 +02005392 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005393 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5394 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5395 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005396 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5397 // Expect to drop this frame, the wait should time out.
5398 ExpectDroppedFrame();
5399
5400 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005401 EXPECT_TRUE_WAIT(
5402 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005403 video_stream_encoder_->Stop();
5404}
5405
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005406TEST_F(VideoStreamEncoderTest,
5407 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5408 webrtc::test::ScopedFieldTrials field_trials(
5409 "WebRTC-Video-QualityScalerSettings/"
5410 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5411 fake_encoder_.SetQualityScaling(false);
5412 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005413 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5414 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005415 const int kWidth = 640;
5416 const int kHeight = 360;
5417
5418 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005419 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005420 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5421 // Frame should not be dropped.
5422 WaitForEncodedFrame(1);
5423
5424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5425 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5426 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5427 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5428 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5429 // Frame should not be dropped.
5430 WaitForEncodedFrame(2);
5431
5432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5433 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5434 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5435 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5436 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5437 // Not dropped since quality scaling is disabled.
5438 WaitForEncodedFrame(3);
5439
5440 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005441 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005442 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5443
5444 video_stream_encoder_->Stop();
5445}
5446
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005447TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005448 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005449 // Set simulcast.
5450 ResetEncoder("VP8", 3, 1, 1, false);
5451 fake_encoder_.SetQualityScaling(true);
5452 const int kWidth = 1280;
5453 const int kHeight = 720;
5454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005455 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5457 // Frame should not be dropped.
5458 WaitForEncodedFrame(1);
5459
5460 // Trigger QVGA "singlecast"
5461 // Update the config.
5462 VideoEncoderConfig video_encoder_config;
5463 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5464 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005465 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005466 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005467 "VP8", /*max qp*/ 56, /*screencast*/ false,
5468 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005469 for (auto& layer : video_encoder_config.simulcast_layers) {
5470 layer.num_temporal_layers = 1;
5471 layer.max_framerate = kDefaultFramerate;
5472 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005473 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005474 video_encoder_config.content_type =
5475 VideoEncoderConfig::ContentType::kRealtimeVideo;
5476
5477 video_encoder_config.simulcast_layers[0].active = true;
5478 video_encoder_config.simulcast_layers[1].active = false;
5479 video_encoder_config.simulcast_layers[2].active = false;
5480
5481 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5482 kMaxPayloadLength);
5483 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5484
5485 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5486 // Frame should not be dropped.
5487 WaitForEncodedFrame(2);
5488
5489 // Trigger HD "singlecast"
5490 video_encoder_config.simulcast_layers[0].active = false;
5491 video_encoder_config.simulcast_layers[1].active = false;
5492 video_encoder_config.simulcast_layers[2].active = true;
5493
5494 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5495 kMaxPayloadLength);
5496 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5497
5498 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5499 // Frame should be dropped because of initial frame drop.
5500 ExpectDroppedFrame();
5501
5502 // Expect the sink_wants to specify a scaled frame.
5503 EXPECT_TRUE_WAIT(
5504 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5505 video_stream_encoder_->Stop();
5506}
5507
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005508TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005509 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005510 // Set simulcast.
5511 ResetEncoder("VP9", 1, 1, 3, false);
5512 fake_encoder_.SetQualityScaling(true);
5513 const int kWidth = 1280;
5514 const int kHeight = 720;
5515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005516 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005517 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5518 // Frame should not be dropped.
5519 WaitForEncodedFrame(1);
5520
5521 // Trigger QVGA "singlecast"
5522 // Update the config.
5523 VideoEncoderConfig video_encoder_config;
5524 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5525 &video_encoder_config);
5526 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5527 vp9_settings.numberOfSpatialLayers = 3;
5528 // Since only one layer is active - automatic resize should be enabled.
5529 vp9_settings.automaticResizeOn = true;
5530 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005531 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005532 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005533 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005534 video_encoder_config.content_type =
5535 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005536 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005537 // which SVC layers are active.
5538 video_encoder_config.simulcast_layers.resize(3);
5539
5540 video_encoder_config.simulcast_layers[0].active = true;
5541 video_encoder_config.simulcast_layers[1].active = false;
5542 video_encoder_config.simulcast_layers[2].active = false;
5543
5544 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5545 kMaxPayloadLength);
5546 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5547
5548 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5549 // Frame should not be dropped.
5550 WaitForEncodedFrame(2);
5551
5552 // Trigger HD "singlecast"
5553 video_encoder_config.simulcast_layers[0].active = false;
5554 video_encoder_config.simulcast_layers[1].active = false;
5555 video_encoder_config.simulcast_layers[2].active = true;
5556
5557 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5558 kMaxPayloadLength);
5559 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5560
5561 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5562 // Frame should be dropped because of initial frame drop.
5563 ExpectDroppedFrame();
5564
5565 // Expect the sink_wants to specify a scaled frame.
5566 EXPECT_TRUE_WAIT(
5567 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5568 video_stream_encoder_->Stop();
5569}
5570
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005571TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005572 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5573 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5574 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5575 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5576 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5577 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5578 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5579 fake_encoder_.SetResolutionBitrateLimits(
5580 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5581
5582 VideoEncoderConfig video_encoder_config;
5583 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5584 &video_encoder_config);
5585 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5586 vp9_settings.numberOfSpatialLayers = 3;
5587 // Since only one layer is active - automatic resize should be enabled.
5588 vp9_settings.automaticResizeOn = true;
5589 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005590 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005591 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005592 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005593 video_encoder_config.content_type =
5594 VideoEncoderConfig::ContentType::kRealtimeVideo;
5595 // Simulcast layers are used to indicate which spatial layers are active.
5596 video_encoder_config.simulcast_layers.resize(3);
5597 video_encoder_config.simulcast_layers[0].active = false;
5598 video_encoder_config.simulcast_layers[1].active = true;
5599 video_encoder_config.simulcast_layers[2].active = false;
5600
5601 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5602 kMaxPayloadLength);
5603 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5604
5605 // The encoder bitrate limits for 360p should be used.
5606 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5607 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005608 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5609 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5610 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5611 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5612 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5613 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005614 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005615 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005616 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005617 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005618
5619 // The encoder bitrate limits for 270p should be used.
5620 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5621 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005622 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5623 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5624 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5625 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5626 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5627 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005628 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005629 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005630 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005631 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005632
5633 video_stream_encoder_->Stop();
5634}
5635
5636TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005637 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5638 VideoEncoderConfig video_encoder_config;
5639 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5640 &video_encoder_config);
5641 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5642 vp9_settings.numberOfSpatialLayers = 3;
5643 // Since only one layer is active - automatic resize should be enabled.
5644 vp9_settings.automaticResizeOn = true;
5645 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005646 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005647 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005648 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005649 video_encoder_config.content_type =
5650 VideoEncoderConfig::ContentType::kRealtimeVideo;
5651 // Simulcast layers are used to indicate which spatial layers are active.
5652 video_encoder_config.simulcast_layers.resize(3);
5653 video_encoder_config.simulcast_layers[0].active = false;
5654 video_encoder_config.simulcast_layers[1].active = true;
5655 video_encoder_config.simulcast_layers[2].active = false;
5656
5657 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5658 kMaxPayloadLength);
5659 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5660
5661 // The default bitrate limits for 360p should be used.
5662 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005663 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5664 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005665 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5666 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005667 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5668 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5669 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5670 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5671 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5672 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005673 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005674 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005675 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005676 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005677
5678 // The default bitrate limits for 270p should be used.
5679 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005680 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5681 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005682 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5683 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005684 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5685 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5686 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5687 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5688 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5689 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005690 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005691 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005692 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005693 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005694
5695 video_stream_encoder_->Stop();
5696}
5697
5698TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5699 webrtc::test::ScopedFieldTrials field_trials(
5700 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5701 VideoEncoderConfig video_encoder_config;
5702 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5703 &video_encoder_config);
5704 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5705 vp9_settings.numberOfSpatialLayers = 3;
5706 // Since only one layer is active - automatic resize should be enabled.
5707 vp9_settings.automaticResizeOn = true;
5708 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005709 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005710 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005711 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005712 video_encoder_config.content_type =
5713 VideoEncoderConfig::ContentType::kRealtimeVideo;
5714 // Simulcast layers are used to indicate which spatial layers are active.
5715 video_encoder_config.simulcast_layers.resize(3);
5716 video_encoder_config.simulcast_layers[0].active = false;
5717 video_encoder_config.simulcast_layers[1].active = true;
5718 video_encoder_config.simulcast_layers[2].active = false;
5719
5720 // Reset encoder for field trials to take effect.
5721 ConfigureEncoder(video_encoder_config.Copy());
5722
5723 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5724 kMaxPayloadLength);
5725 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5726
5727 // The default bitrate limits for 360p should not be used.
5728 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005729 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5730 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005731 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5732 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005733 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5734 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5735 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5736 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5737 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5738 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005739 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005740 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005741
5742 video_stream_encoder_->Stop();
5743}
5744
5745TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5746 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5747 /*num_spatial_layers=*/1, /*screenshare=*/false);
5748
5749 // The default singlecast bitrate limits for 720p should not be used.
5750 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005751 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5752 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005753 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5754 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005755 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5756 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5757 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5758 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5759 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5760 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005761 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005762 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005763
5764 video_stream_encoder_->Stop();
5765}
5766
5767TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005768 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5769 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5770 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5771 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5772 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5773 fake_encoder_.SetResolutionBitrateLimits(
5774 {kEncoderLimits180p, kEncoderLimits720p});
5775
5776 VideoEncoderConfig video_encoder_config;
5777 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5778 &video_encoder_config);
5779 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5780 vp9_settings.numberOfSpatialLayers = 3;
5781 // Since only one layer is active - automatic resize should be enabled.
5782 vp9_settings.automaticResizeOn = true;
5783 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005784 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005785 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005786 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005787 video_encoder_config.content_type =
5788 VideoEncoderConfig::ContentType::kRealtimeVideo;
5789 // Simulcast layers are used to indicate which spatial layers are active.
5790 video_encoder_config.simulcast_layers.resize(3);
5791 video_encoder_config.simulcast_layers[0].active = true;
5792 video_encoder_config.simulcast_layers[1].active = false;
5793 video_encoder_config.simulcast_layers[2].active = false;
5794
5795 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5796 kMaxPayloadLength);
5797 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5798
5799 // Limits not applied on lowest stream, limits for 180p should not be used.
5800 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5801 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5803 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5804 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5805 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5806 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5807 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005808 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005809 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005810 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005811 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005812
5813 video_stream_encoder_->Stop();
5814}
5815
5816TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005817 InitialFrameDropActivatesWhenResolutionIncreases) {
5818 const int kWidth = 640;
5819 const int kHeight = 360;
5820
5821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005823 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5824 // Frame should not be dropped.
5825 WaitForEncodedFrame(1);
5826
5827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005828 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005829 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5830 // Frame should not be dropped, bitrate not too low for frame.
5831 WaitForEncodedFrame(2);
5832
5833 // Incoming resolution increases.
5834 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5835 // Expect to drop this frame, bitrate too low for frame.
5836 ExpectDroppedFrame();
5837
5838 // Expect the sink_wants to specify a scaled frame.
5839 EXPECT_TRUE_WAIT(
5840 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5841 video_stream_encoder_->Stop();
5842}
5843
5844TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5845 const int kWidth = 640;
5846 const int kHeight = 360;
5847 // So that quality scaling doesn't happen by itself.
5848 fake_encoder_.SetQp(kQpHigh);
5849
5850 AdaptingFrameForwarder source(&time_controller_);
5851 source.set_adaptation_enabled(true);
5852 video_stream_encoder_->SetSource(
5853 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5854
5855 int timestamp = 1;
5856
5857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005859 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5860 WaitForEncodedFrame(timestamp);
5861 timestamp += 9000;
5862 // Long pause to disable all first BWE drop logic.
5863 AdvanceTime(TimeDelta::Millis(1000));
5864
5865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005866 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005867 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5868 // Not dropped frame, as initial frame drop is disabled by now.
5869 WaitForEncodedFrame(timestamp);
5870 timestamp += 9000;
5871 AdvanceTime(TimeDelta::Millis(100));
5872
5873 // Quality adaptation down.
5874 video_stream_encoder_->TriggerQualityLow();
5875
5876 // Adaptation has an effect.
5877 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5878 5000);
5879
5880 // Frame isn't dropped as initial frame dropper is disabled.
5881 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5882 WaitForEncodedFrame(timestamp);
5883 timestamp += 9000;
5884 AdvanceTime(TimeDelta::Millis(100));
5885
5886 // Quality adaptation up.
5887 video_stream_encoder_->TriggerQualityHigh();
5888
5889 // Adaptation has an effect.
5890 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5891 5000);
5892
5893 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5894 // Frame should not be dropped, as initial framedropper is off.
5895 WaitForEncodedFrame(timestamp);
5896
5897 video_stream_encoder_->Stop();
5898}
5899
Åsa Persson7f354f82021-02-04 15:52:15 +01005900TEST_F(VideoStreamEncoderTest,
5901 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5902 const int kMinStartBps360p = 222000;
5903 fake_encoder_.SetResolutionBitrateLimits(
5904 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5905 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5906 800000)});
5907
5908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5909 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5910 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5911 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5912 0, 0, 0);
5913 // Frame should not be dropped, bitrate not too low for frame.
5914 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5915 WaitForEncodedFrame(1);
5916
5917 // Incoming resolution increases, initial frame drop activates.
5918 // Frame should be dropped, link allocation too low for frame.
5919 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5920 ExpectDroppedFrame();
5921
5922 // Expect sink_wants to specify a scaled frame.
5923 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5924 5000);
5925 video_stream_encoder_->Stop();
5926}
5927
5928TEST_F(VideoStreamEncoderTest,
5929 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5930 const int kMinStartBps360p = 222000;
5931 fake_encoder_.SetResolutionBitrateLimits(
5932 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5933 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5934 800000)});
5935
5936 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5937 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5938 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5939 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5940 0, 0, 0);
5941 // Frame should not be dropped, bitrate not too low for frame.
5942 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5943 WaitForEncodedFrame(1);
5944
5945 // Incoming resolution increases, initial frame drop activates.
5946 // Frame should be dropped, link allocation not too low for frame.
5947 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5948 WaitForEncodedFrame(2);
5949
5950 video_stream_encoder_->Stop();
5951}
5952
Åsa Perssone644a032019-11-08 15:56:00 +01005953TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5954 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02005955 "WebRTC-Video-QualityRampupSettings/"
5956 "min_pixels:921600,min_duration_ms:2000/");
5957
5958 const int kWidth = 1280;
5959 const int kHeight = 720;
5960 const int kFps = 10;
5961 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01005962
5963 // Reset encoder for field trials to take effect.
5964 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02005965 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02005966 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005967 ConfigureEncoder(std::move(config));
5968 fake_encoder_.SetQp(kQpLow);
5969
5970 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005971 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005972 source.set_adaptation_enabled(true);
5973 video_stream_encoder_->SetSource(&source,
5974 DegradationPreference::MAINTAIN_FRAMERATE);
5975
5976 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02005977 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02005978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005979 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005980
5981 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02005982 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01005983 int64_t timestamp_ms = kFrameIntervalMs;
5984 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5985 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005986 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5987 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005988
5989 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5991 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005992
Artem Titovab30d722021-07-27 16:22:11 +02005993 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02005994 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01005995 for (size_t i = 1; i <= 10; i++) {
5996 timestamp_ms += kFrameIntervalMs;
5997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5998 WaitForEncodedFrame(timestamp_ms);
5999 }
Åsa Persson06defc42021-09-10 15:28:48 +02006000
6001 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6002 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6003 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6004 timestamp_ms += kFrameIntervalMs;
6005 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6006 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006007 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6008 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6009
Åsa Persson06defc42021-09-10 15:28:48 +02006010 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006011 timestamp_ms += kFrameIntervalMs;
6012 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6013 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006014 // The ramp-up code involves the adaptation queue, give it time to execute.
6015 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006016 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006017 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006018
6019 // Frame should not be adapted.
6020 timestamp_ms += kFrameIntervalMs;
6021 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6022 WaitForEncodedFrame(kWidth, kHeight);
6023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6024
6025 video_stream_encoder_->Stop();
6026}
6027
mflodmancc3d4422017-08-03 08:27:51 -07006028TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006029 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006030 webrtc::test::ScopedFieldTrials field_trials(
6031 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006032 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006033 source.set_adaptation_enabled(true);
6034 video_stream_encoder_->SetSource(&source,
6035 DegradationPreference::MAINTAIN_FRAMERATE);
6036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006037 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006038 fake_encoder_.SetQp(kQpHigh + 1);
6039 const int kWidth = 1280;
6040 const int kHeight = 720;
6041 const int64_t kFrameIntervalMs = 100;
6042 int64_t timestamp_ms = kFrameIntervalMs;
6043 for (size_t i = 1; i <= 100; i++) {
6044 timestamp_ms += kFrameIntervalMs;
6045 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6046 WaitForEncodedFrame(timestamp_ms);
6047 }
6048 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6049 // for the first time.
6050 // TODO(eshr): We should avoid these waits by using threads with simulated
6051 // time.
6052 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6053 2000 * 2.5 * 2);
6054 timestamp_ms += kFrameIntervalMs;
6055 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6056 WaitForEncodedFrame(timestamp_ms);
6057 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6058 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6059 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6060
6061 // Disable Quality scaling by turning off scaler on the encoder and
6062 // reconfiguring.
6063 fake_encoder_.SetQualityScaling(false);
6064 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6065 kMaxPayloadLength);
6066 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006067 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006068 // Since we turned off the quality scaler, the adaptations made by it are
6069 // removed.
6070 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6071 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6072
6073 video_stream_encoder_->Stop();
6074}
6075
6076TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006077 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6078 const int kTooSmallWidth = 10;
6079 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006081 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006082
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006083 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006084 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006085 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006086 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006087 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006088 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6089
6090 // Trigger adapt down, too small frame, expect no change.
6091 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006092 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006093 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006094 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006095 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6096 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6097
mflodmancc3d4422017-08-03 08:27:51 -07006098 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006099}
6100
mflodmancc3d4422017-08-03 08:27:51 -07006101TEST_F(VideoStreamEncoderTest,
6102 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006103 const int kTooSmallWidth = 10;
6104 const int kTooSmallHeight = 10;
6105 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006107 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006108
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006109 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006110 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006111 video_stream_encoder_->SetSource(&source,
6112 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006113 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006114 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6116
6117 // Trigger adapt down, expect limited framerate.
6118 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006119 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006120 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006121 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006122 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6123 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6124 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6125
6126 // Trigger adapt down, too small frame, expect no change.
6127 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006128 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006129 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006130 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006131 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6132 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6133 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6134
mflodmancc3d4422017-08-03 08:27:51 -07006135 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006136}
6137
mflodmancc3d4422017-08-03 08:27:51 -07006138TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006139 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006140 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006141 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006142 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006143 const int kFrameWidth = 1280;
6144 const int kFrameHeight = 720;
6145 video_source_.IncomingCapturedFrame(
6146 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006147 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006148 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006149}
6150
sprangb1ca0732017-02-01 08:38:12 -08006151// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006152TEST_F(VideoStreamEncoderTest,
6153 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006155 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006156
6157 const int kFrameWidth = 1280;
6158 const int kFrameHeight = 720;
6159 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006160 // requested by
6161 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006162 video_source_.set_adaptation_enabled(true);
6163
6164 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006165 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006166 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006167
6168 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006169 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006170 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006171 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006172 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006173
asaperssonfab67072017-04-04 05:51:49 -07006174 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006175 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006176 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006177 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006178 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006179
mflodmancc3d4422017-08-03 08:27:51 -07006180 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006181}
sprangfe627f32017-03-29 08:24:59 -07006182
mflodmancc3d4422017-08-03 08:27:51 -07006183TEST_F(VideoStreamEncoderTest,
6184 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006185 const int kFrameWidth = 1280;
6186 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006187
Henrik Boström381d1092020-05-12 18:49:07 +02006188 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006189 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006190 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006191 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006192 video_source_.set_adaptation_enabled(true);
6193
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006194 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006195
6196 video_source_.IncomingCapturedFrame(
6197 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006198 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006199
6200 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006201 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006202
6203 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006204 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006205 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006206 video_source_.IncomingCapturedFrame(
6207 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006208 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006209 }
6210
6211 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006212 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006213 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006214 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006215 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006216 video_source_.IncomingCapturedFrame(
6217 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006218 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006219 ++num_frames_dropped;
6220 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006221 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006222 }
6223 }
6224
sprang4847ae62017-06-27 07:06:52 -07006225 // Add some slack to account for frames dropped by the frame dropper.
6226 const int kErrorMargin = 1;
6227 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006228 kErrorMargin);
6229
6230 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006231 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006232 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006233 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006234 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006235 video_source_.IncomingCapturedFrame(
6236 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006237 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006238 ++num_frames_dropped;
6239 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006240 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006241 }
6242 }
sprang4847ae62017-06-27 07:06:52 -07006243 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006244 kErrorMargin);
6245
6246 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006247 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006248 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006249 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006250 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006251 video_source_.IncomingCapturedFrame(
6252 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006253 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006254 ++num_frames_dropped;
6255 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006256 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006257 }
6258 }
sprang4847ae62017-06-27 07:06:52 -07006259 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006260 kErrorMargin);
6261
6262 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006263 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006264 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006265 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006266 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006267 video_source_.IncomingCapturedFrame(
6268 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006269 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006270 ++num_frames_dropped;
6271 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006272 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006273 }
6274 }
6275 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6276
mflodmancc3d4422017-08-03 08:27:51 -07006277 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006278}
6279
mflodmancc3d4422017-08-03 08:27:51 -07006280TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006281 const int kFramerateFps = 5;
6282 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006283 const int kFrameWidth = 1280;
6284 const int kFrameHeight = 720;
6285
sprang4847ae62017-06-27 07:06:52 -07006286 // Reconfigure encoder with two temporal layers and screensharing, which will
6287 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006288 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006289
Henrik Boström381d1092020-05-12 18:49:07 +02006290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006291 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006292 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006293 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006294 video_source_.set_adaptation_enabled(true);
6295
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006296 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006297
6298 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006299 rtc::VideoSinkWants last_wants;
6300 do {
6301 last_wants = video_source_.sink_wants();
6302
sprangc5d62e22017-04-02 23:53:04 -07006303 // Insert frames to get a new fps estimate...
6304 for (int j = 0; j < kFramerateFps; ++j) {
6305 video_source_.IncomingCapturedFrame(
6306 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006307 if (video_source_.last_sent_width()) {
6308 sink_.WaitForEncodedFrame(timestamp_ms);
6309 }
sprangc5d62e22017-04-02 23:53:04 -07006310 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006311 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006312 }
6313 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006314 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006315 } while (video_source_.sink_wants().max_framerate_fps <
6316 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006317
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006318 EXPECT_THAT(video_source_.sink_wants(),
6319 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006320
mflodmancc3d4422017-08-03 08:27:51 -07006321 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006322}
asaperssonf7e294d2017-06-13 23:25:22 -07006323
mflodmancc3d4422017-08-03 08:27:51 -07006324TEST_F(VideoStreamEncoderTest,
6325 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006326 const int kWidth = 1280;
6327 const int kHeight = 720;
6328 const int64_t kFrameIntervalMs = 150;
6329 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006331 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006332
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006333 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006334 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006335 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006336 video_stream_encoder_->SetSource(&source,
6337 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006338 timestamp_ms += kFrameIntervalMs;
6339 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006340 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006341 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006342 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6344 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6345
6346 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006347 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006348 timestamp_ms += kFrameIntervalMs;
6349 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006350 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006351 EXPECT_THAT(source.sink_wants(),
6352 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006353 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6354 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6355 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6356
6357 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006358 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006359 timestamp_ms += kFrameIntervalMs;
6360 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006361 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006362 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006363 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6364 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6365 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6366
6367 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006368 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006369 timestamp_ms += kFrameIntervalMs;
6370 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006371 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006372 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006373 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6374 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6375 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6376
6377 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006378 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006379 timestamp_ms += kFrameIntervalMs;
6380 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006381 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006382 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006383 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6384 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6385 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6386
6387 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006388 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006389 timestamp_ms += kFrameIntervalMs;
6390 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006391 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006392 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006393 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6395 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6396
6397 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006398 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006399 timestamp_ms += kFrameIntervalMs;
6400 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006401 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006402 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006403 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6404 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6405 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6406
6407 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006408 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006409 timestamp_ms += kFrameIntervalMs;
6410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006411 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006412 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006413 rtc::VideoSinkWants last_wants = source.sink_wants();
6414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6416 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6417
6418 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006419 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006420 timestamp_ms += kFrameIntervalMs;
6421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006422 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006423 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6426 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6427
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006428 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006429 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006430 timestamp_ms += kFrameIntervalMs;
6431 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006432 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006433 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6435 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6436 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6437
6438 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006439 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006440 timestamp_ms += kFrameIntervalMs;
6441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006442 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006443 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006444 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6445 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6446 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6447
6448 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006449 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006450 timestamp_ms += kFrameIntervalMs;
6451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006452 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006453 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6455 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6456 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6457
6458 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006459 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006460 timestamp_ms += kFrameIntervalMs;
6461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006462 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006463 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6465 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6466 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6467
6468 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006469 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006470 timestamp_ms += kFrameIntervalMs;
6471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006473 EXPECT_THAT(source.sink_wants(), FpsMax());
6474 EXPECT_EQ(source.sink_wants().max_pixel_count,
6475 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6478 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6479
6480 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006481 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006482 timestamp_ms += kFrameIntervalMs;
6483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006484 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006485 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006486 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6488 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6489
Åsa Persson30ab0152019-08-27 12:22:33 +02006490 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006491 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006492 timestamp_ms += kFrameIntervalMs;
6493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006494 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006495 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006496 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6499 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6500
6501 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006502 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006503 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006504 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6505
mflodmancc3d4422017-08-03 08:27:51 -07006506 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006507}
6508
mflodmancc3d4422017-08-03 08:27:51 -07006509TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006510 const int kWidth = 1280;
6511 const int kHeight = 720;
6512 const int64_t kFrameIntervalMs = 150;
6513 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006516
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006517 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006518 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006519 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006520 video_stream_encoder_->SetSource(&source,
6521 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006522 timestamp_ms += kFrameIntervalMs;
6523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006524 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6528 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6529 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6530 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6531 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6532
6533 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006534 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006538 EXPECT_THAT(source.sink_wants(),
6539 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6542 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6543 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6544 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6545 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6546
6547 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006548 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006549 timestamp_ms += kFrameIntervalMs;
6550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6555 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6557 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6558 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6569 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6570 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6571 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
Evan Shrubsole64469032020-06-11 10:45:29 +02006573 // Trigger cpu adapt up, expect no change since QP is most limited.
6574 {
6575 // Store current sink wants since we expect no change and if there is no
6576 // change then last_wants() is not updated.
6577 auto previous_sink_wants = source.sink_wants();
6578 video_stream_encoder_->TriggerCpuUnderuse();
6579 timestamp_ms += kFrameIntervalMs;
6580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6581 WaitForEncodedFrame(timestamp_ms);
6582 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6583 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6584 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6585 }
6586
6587 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6588 video_stream_encoder_->TriggerQualityHigh();
6589 timestamp_ms += kFrameIntervalMs;
6590 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6591 WaitForEncodedFrame(timestamp_ms);
6592 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6594 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6595 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6596 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6597 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6598 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6599
6600 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6601 // expect increased resolution (960x540@30fps).
6602 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006603 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006604 timestamp_ms += kFrameIntervalMs;
6605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006606 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006607 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6609 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6610 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6612 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006613 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006614
Evan Shrubsole64469032020-06-11 10:45:29 +02006615 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6616 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006617 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006618 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006619 timestamp_ms += kFrameIntervalMs;
6620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006621 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006623 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6625 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6628 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006629 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006630
6631 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006632 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006633 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006634 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006635 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006636
mflodmancc3d4422017-08-03 08:27:51 -07006637 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006638}
6639
mflodmancc3d4422017-08-03 08:27:51 -07006640TEST_F(VideoStreamEncoderTest,
6641 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006642 const int kWidth = 640;
6643 const int kHeight = 360;
6644 const int kFpsLimit = 15;
6645 const int64_t kFrameIntervalMs = 150;
6646 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006648 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006649
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006650 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006651 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006652 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006653 video_stream_encoder_->SetSource(&source,
6654 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006655 timestamp_ms += kFrameIntervalMs;
6656 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006657 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006658 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006659 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6661 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6663 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6664 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6665
6666 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006667 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006668 timestamp_ms += kFrameIntervalMs;
6669 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006670 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006671 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006672 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6675 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6676 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6677 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6678
6679 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006680 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006681 timestamp_ms += kFrameIntervalMs;
6682 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006683 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006684 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006685 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006686 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006687 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6688 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6689 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6690 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6691
Evan Shrubsole64469032020-06-11 10:45:29 +02006692 // Trigger cpu adapt up, expect no change because quality is most limited.
6693 {
6694 auto previous_sink_wants = source.sink_wants();
6695 // Store current sink wants since we expect no change ind if there is no
6696 // change then last__wants() is not updated.
6697 video_stream_encoder_->TriggerCpuUnderuse();
6698 timestamp_ms += kFrameIntervalMs;
6699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6700 WaitForEncodedFrame(timestamp_ms);
6701 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6702 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6703 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6704 }
6705
6706 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6707 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006708 timestamp_ms += kFrameIntervalMs;
6709 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006710 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006711 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006712 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6713 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6714 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006715 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6717 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006718
Evan Shrubsole64469032020-06-11 10:45:29 +02006719 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006720 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006721 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006722 timestamp_ms += kFrameIntervalMs;
6723 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006724 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006725 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006726 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6727 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6730 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006731 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006732
6733 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006734 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006735 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006736 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006737 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006738
mflodmancc3d4422017-08-03 08:27:51 -07006739 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006740}
6741
mflodmancc3d4422017-08-03 08:27:51 -07006742TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006743 const int kFrameWidth = 1920;
6744 const int kFrameHeight = 1080;
6745 // 3/4 of 1920.
6746 const int kAdaptedFrameWidth = 1440;
6747 // 3/4 of 1080 rounded down to multiple of 4.
6748 const int kAdaptedFrameHeight = 808;
6749 const int kFramerate = 24;
6750
Henrik Boström381d1092020-05-12 18:49:07 +02006751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006753 // Trigger reconfigure encoder (without resetting the entire instance).
6754 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006755 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6756 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006757 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006758 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006759 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006760 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006761 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006762 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006763
6764 video_source_.set_adaptation_enabled(true);
6765
6766 video_source_.IncomingCapturedFrame(
6767 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006768 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006769
6770 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006771 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006772 video_source_.IncomingCapturedFrame(
6773 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006774 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006775
mflodmancc3d4422017-08-03 08:27:51 -07006776 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006777}
6778
mflodmancc3d4422017-08-03 08:27:51 -07006779TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006780 const int kFrameWidth = 1280;
6781 const int kFrameHeight = 720;
6782 const int kLowFps = 2;
6783 const int kHighFps = 30;
6784
Henrik Boström381d1092020-05-12 18:49:07 +02006785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006786 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006787
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006788 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006789 max_framerate_ = kLowFps;
6790
6791 // Insert 2 seconds of 2fps video.
6792 for (int i = 0; i < kLowFps * 2; ++i) {
6793 video_source_.IncomingCapturedFrame(
6794 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6795 WaitForEncodedFrame(timestamp_ms);
6796 timestamp_ms += 1000 / kLowFps;
6797 }
6798
6799 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006802 video_source_.IncomingCapturedFrame(
6803 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6804 WaitForEncodedFrame(timestamp_ms);
6805 timestamp_ms += 1000 / kLowFps;
6806
6807 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6808
6809 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006810 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006811 const int kFrameIntervalMs = 1000 / kHighFps;
6812 max_framerate_ = kHighFps;
6813 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6814 video_source_.IncomingCapturedFrame(
6815 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6816 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6817 // be dropped if the encoder hans't been updated with the new higher target
6818 // framerate yet, causing it to overshoot the target bitrate and then
6819 // suffering the wrath of the media optimizer.
6820 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6821 timestamp_ms += kFrameIntervalMs;
6822 }
6823
6824 // Don expect correct measurement just yet, but it should be higher than
6825 // before.
6826 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6827
mflodmancc3d4422017-08-03 08:27:51 -07006828 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006829}
6830
mflodmancc3d4422017-08-03 08:27:51 -07006831TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006832 const int kFrameWidth = 1280;
6833 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006834 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006835 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006836 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006837
Henrik Boström381d1092020-05-12 18:49:07 +02006838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006839 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006841
6842 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006843 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006844 video_source_.IncomingCapturedFrame(
6845 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6846 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006847 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006848
6849 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006851 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006852
6853 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006854 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006855 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006856
Per Kjellanderdcef6412020-10-07 15:09:05 +02006857 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006858 video_source_.IncomingCapturedFrame(
6859 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6860 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006861 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006862
mflodmancc3d4422017-08-03 08:27:51 -07006863 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006864}
ilnik6b826ef2017-06-16 06:53:48 -07006865
Niels Möller4db138e2018-04-19 09:04:13 +02006866TEST_F(VideoStreamEncoderTest,
6867 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6868 const int kFrameWidth = 1280;
6869 const int kFrameHeight = 720;
6870 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006873 video_source_.IncomingCapturedFrame(
6874 CreateFrame(1, kFrameWidth, kFrameHeight));
6875 WaitForEncodedFrame(1);
6876 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6877 .low_encode_usage_threshold_percent,
6878 default_options.low_encode_usage_threshold_percent);
6879 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6880 .high_encode_usage_threshold_percent,
6881 default_options.high_encode_usage_threshold_percent);
6882 video_stream_encoder_->Stop();
6883}
6884
6885TEST_F(VideoStreamEncoderTest,
6886 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6887 const int kFrameWidth = 1280;
6888 const int kFrameHeight = 720;
6889 CpuOveruseOptions hardware_options;
6890 hardware_options.low_encode_usage_threshold_percent = 150;
6891 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006892 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006893
Henrik Boström381d1092020-05-12 18:49:07 +02006894 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006895 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006896 video_source_.IncomingCapturedFrame(
6897 CreateFrame(1, kFrameWidth, kFrameHeight));
6898 WaitForEncodedFrame(1);
6899 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6900 .low_encode_usage_threshold_percent,
6901 hardware_options.low_encode_usage_threshold_percent);
6902 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6903 .high_encode_usage_threshold_percent,
6904 hardware_options.high_encode_usage_threshold_percent);
6905 video_stream_encoder_->Stop();
6906}
6907
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006908TEST_F(VideoStreamEncoderTest,
6909 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6910 const int kFrameWidth = 1280;
6911 const int kFrameHeight = 720;
6912
6913 const CpuOveruseOptions default_options;
6914 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006915 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006916 video_source_.IncomingCapturedFrame(
6917 CreateFrame(1, kFrameWidth, kFrameHeight));
6918 WaitForEncodedFrame(1);
6919 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6920 .low_encode_usage_threshold_percent,
6921 default_options.low_encode_usage_threshold_percent);
6922 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6923 .high_encode_usage_threshold_percent,
6924 default_options.high_encode_usage_threshold_percent);
6925
6926 CpuOveruseOptions hardware_options;
6927 hardware_options.low_encode_usage_threshold_percent = 150;
6928 hardware_options.high_encode_usage_threshold_percent = 200;
6929 fake_encoder_.SetIsHardwareAccelerated(true);
6930
6931 video_source_.IncomingCapturedFrame(
6932 CreateFrame(2, kFrameWidth, kFrameHeight));
6933 WaitForEncodedFrame(2);
6934
6935 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6936 .low_encode_usage_threshold_percent,
6937 hardware_options.low_encode_usage_threshold_percent);
6938 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6939 .high_encode_usage_threshold_percent,
6940 hardware_options.high_encode_usage_threshold_percent);
6941
6942 video_stream_encoder_->Stop();
6943}
6944
Niels Möller6bb5ab92019-01-11 11:11:10 +01006945TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6946 const int kFrameWidth = 320;
6947 const int kFrameHeight = 240;
6948 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02006949 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006950 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6951
Henrik Boström381d1092020-05-12 18:49:07 +02006952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006953 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006954
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006955 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006956 max_framerate_ = kFps;
6957
6958 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6959 fake_encoder_.SimulateOvershoot(1.0);
6960 int num_dropped = 0;
6961 for (int i = 0; i < kNumFramesInRun; ++i) {
6962 video_source_.IncomingCapturedFrame(
6963 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6964 // Wait up to two frame durations for a frame to arrive.
6965 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6966 ++num_dropped;
6967 }
6968 timestamp_ms += 1000 / kFps;
6969 }
6970
Erik Språnga8d48ab2019-02-08 14:17:40 +01006971 // Framerate should be measured to be near the expected target rate.
6972 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6973
6974 // Frame drops should be within 5% of expected 0%.
6975 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006976
6977 // Make encoder produce frames at double the expected bitrate during 3 seconds
6978 // of video, verify number of drops. Rate needs to be slightly changed in
6979 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006980 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006981 const RateControlSettings trials =
6982 RateControlSettings::ParseFromFieldTrials();
6983 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006984 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006985 // frame dropping since the adjuter will try to just lower the target
6986 // bitrate rather than drop frames. If network headroom can be used, it
6987 // doesn't push back as hard so we don't need quite as much overshoot.
6988 // These numbers are unfortunately a bit magical but there's not trivial
6989 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01006990 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01006991 }
6992 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006994 kTargetBitrate + DataRate::KilobitsPerSec(1),
6995 kTargetBitrate + DataRate::KilobitsPerSec(1),
6996 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006997 num_dropped = 0;
6998 for (int i = 0; i < kNumFramesInRun; ++i) {
6999 video_source_.IncomingCapturedFrame(
7000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7001 // Wait up to two frame durations for a frame to arrive.
7002 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7003 ++num_dropped;
7004 }
7005 timestamp_ms += 1000 / kFps;
7006 }
7007
Henrik Boström381d1092020-05-12 18:49:07 +02007008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007009 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007010
7011 // Target framerate should be still be near the expected target, despite
7012 // the frame drops.
7013 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7014
7015 // Frame drops should be within 5% of expected 50%.
7016 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007017
7018 video_stream_encoder_->Stop();
7019}
7020
7021TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7022 const int kFrameWidth = 320;
7023 const int kFrameHeight = 240;
7024 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007025 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007026
7027 ASSERT_GT(max_framerate_, kActualInputFps);
7028
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007029 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007030 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007033
7034 // Insert 3 seconds of video, with an input fps lower than configured max.
7035 for (int i = 0; i < kActualInputFps * 3; ++i) {
7036 video_source_.IncomingCapturedFrame(
7037 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7038 // Wait up to two frame durations for a frame to arrive.
7039 WaitForEncodedFrame(timestamp_ms);
7040 timestamp_ms += 1000 / kActualInputFps;
7041 }
7042
7043 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7044
7045 video_stream_encoder_->Stop();
7046}
7047
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007048TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007049 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007051 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007052
7053 fake_encoder_.BlockNextEncode();
7054 video_source_.IncomingCapturedFrame(
7055 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7056 WaitForEncodedFrame(1);
7057 // On the very first frame full update should be forced.
7058 rect = fake_encoder_.GetLastUpdateRect();
7059 EXPECT_EQ(rect.offset_x, 0);
7060 EXPECT_EQ(rect.offset_y, 0);
7061 EXPECT_EQ(rect.height, codec_height_);
7062 EXPECT_EQ(rect.width, codec_width_);
7063 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7064 // call to ContinueEncode.
7065 video_source_.IncomingCapturedFrame(
7066 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7067 ExpectDroppedFrame();
7068 video_source_.IncomingCapturedFrame(
7069 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7070 ExpectDroppedFrame();
7071 fake_encoder_.ContinueEncode();
7072 WaitForEncodedFrame(3);
7073 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7074 rect = fake_encoder_.GetLastUpdateRect();
7075 EXPECT_EQ(rect.offset_x, 1);
7076 EXPECT_EQ(rect.offset_y, 0);
7077 EXPECT_EQ(rect.width, 10);
7078 EXPECT_EQ(rect.height, 1);
7079
7080 video_source_.IncomingCapturedFrame(
7081 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7082 WaitForEncodedFrame(4);
7083 // Previous frame was encoded, so no accumulation should happen.
7084 rect = fake_encoder_.GetLastUpdateRect();
7085 EXPECT_EQ(rect.offset_x, 0);
7086 EXPECT_EQ(rect.offset_y, 0);
7087 EXPECT_EQ(rect.width, 1);
7088 EXPECT_EQ(rect.height, 1);
7089
7090 video_stream_encoder_->Stop();
7091}
7092
Erik Språngd7329ca2019-02-21 21:19:53 +01007093TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007095 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007096
7097 // First frame is always keyframe.
7098 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7099 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007100 EXPECT_THAT(
7101 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007102 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007103
7104 // Insert delta frame.
7105 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7106 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007107 EXPECT_THAT(
7108 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007109 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007110
7111 // Request next frame be a key-frame.
7112 video_stream_encoder_->SendKeyFrame();
7113 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7114 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007115 EXPECT_THAT(
7116 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007117 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007118
7119 video_stream_encoder_->Stop();
7120}
7121
7122TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7123 // Setup simulcast with three streams.
7124 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007126 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7127 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007128 // Wait for all three layers before triggering event.
7129 sink_.SetNumExpectedLayers(3);
7130
7131 // First frame is always keyframe.
7132 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7133 WaitForEncodedFrame(1);
7134 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007135 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7136 VideoFrameType::kVideoFrameKey,
7137 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007138
7139 // Insert delta frame.
7140 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7141 WaitForEncodedFrame(2);
7142 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007143 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7144 VideoFrameType::kVideoFrameDelta,
7145 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007146
7147 // Request next frame be a key-frame.
7148 // Only first stream is configured to produce key-frame.
7149 video_stream_encoder_->SendKeyFrame();
7150 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7151 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007152
7153 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7154 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007155 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007156 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007157 VideoFrameType::kVideoFrameKey,
7158 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007159
7160 video_stream_encoder_->Stop();
7161}
7162
7163TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7164 // Configure internal source factory and setup test again.
7165 encoder_factory_.SetHasInternalSource(true);
7166 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007168 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007169
7170 // Call encoder directly, simulating internal source where encoded frame
7171 // callback in VideoStreamEncoder is called despite no OnFrame().
7172 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7173 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007174 EXPECT_THAT(
7175 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007176 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007177
Niels Möller8f7ce222019-03-21 15:43:58 +01007178 const std::vector<VideoFrameType> kDeltaFrame = {
7179 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007180 // Need to set timestamp manually since manually for injected frame.
7181 VideoFrame frame = CreateFrame(101, nullptr);
7182 frame.set_timestamp(101);
7183 fake_encoder_.InjectFrame(frame, false);
7184 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007185 EXPECT_THAT(
7186 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007187 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007188
7189 // Request key-frame. The forces a dummy frame down into the encoder.
7190 fake_encoder_.ExpectNullFrame();
7191 video_stream_encoder_->SendKeyFrame();
7192 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007193 EXPECT_THAT(
7194 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007195 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007196
7197 video_stream_encoder_->Stop();
7198}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007199
7200TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7201 // Configure internal source factory and setup test again.
7202 encoder_factory_.SetHasInternalSource(true);
7203 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007204 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007205 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007206
7207 int64_t timestamp = 1;
7208 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007209 image.capture_time_ms_ = ++timestamp;
7210 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7211 const int64_t kEncodeFinishDelayMs = 10;
7212 image.timing_.encode_start_ms = timestamp;
7213 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007214 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007215 // Wait for frame without incrementing clock.
7216 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7217 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7218 // capture timestamp should be kEncodeFinishDelayMs in the past.
7219 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007220 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007221
7222 video_stream_encoder_->Stop();
7223}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007224
7225TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007226 // SPS contains VUI with restrictions on the maximum number of reordered
7227 // pictures, there is no need to rewrite the bitstream to enable faster
7228 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007229 ResetEncoder("H264", 1, 1, 1, false);
7230
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007231 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007232 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007233 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007234
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007235 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007236 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007237
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007238 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7239 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007240
7241 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007242 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007243
7244 video_stream_encoder_->Stop();
7245}
7246
7247TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007248 // SPS does not contain VUI, the bitstream is will be rewritten with added
7249 // VUI with restrictions on the maximum number of reordered pictures to
7250 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007251 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7252 0x00, 0x00, 0x03, 0x03, 0xF4,
7253 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007254 ResetEncoder("H264", 1, 1, 1, false);
7255
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007257 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007259
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007260 fake_encoder_.SetEncodedImageData(
7261 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007262
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007263 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7264 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007265
7266 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007267 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007268
7269 video_stream_encoder_->Stop();
7270}
7271
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007272TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7273 const int kFrameWidth = 1280;
7274 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007275 const DataRate kTargetBitrate =
7276 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007277
Henrik Boström381d1092020-05-12 18:49:07 +02007278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007279 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007280 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7281
7282 // Insert a first video frame. It should be dropped because of downscale in
7283 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007284 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007285 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7286 frame.set_rotation(kVideoRotation_270);
7287 video_source_.IncomingCapturedFrame(frame);
7288
7289 ExpectDroppedFrame();
7290
7291 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007292 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007293 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7294 frame.set_rotation(kVideoRotation_90);
7295 video_source_.IncomingCapturedFrame(frame);
7296
7297 WaitForEncodedFrame(timestamp_ms);
7298 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7299
7300 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007301 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007302 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7303 frame.set_rotation(kVideoRotation_180);
7304 video_source_.IncomingCapturedFrame(frame);
7305
7306 WaitForEncodedFrame(timestamp_ms);
7307 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7308
7309 video_stream_encoder_->Stop();
7310}
7311
Erik Språng5056af02019-09-02 15:53:11 +02007312TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7313 const int kFrameWidth = 320;
7314 const int kFrameHeight = 180;
7315
7316 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007318 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7319 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7320 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007321 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007322 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007323 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007324
7325 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007326 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007327 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7328 frame.set_rotation(kVideoRotation_270);
7329 video_source_.IncomingCapturedFrame(frame);
7330 WaitForEncodedFrame(timestamp_ms);
7331
7332 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007333 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007334 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7335 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007337 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007338 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007339 /*link_allocation=*/target_rate,
7340 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007341 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007342 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007343 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7344
7345 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7346 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7347 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007348 DataRate allocation_sum =
7349 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007350 EXPECT_EQ(min_rate, allocation_sum);
7351 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7352
7353 video_stream_encoder_->Stop();
7354}
7355
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007356TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007357 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007358 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007359 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007360 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007361 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7362 WaitForEncodedFrame(1);
7363
7364 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7365 ASSERT_TRUE(prev_rate_settings.has_value());
7366 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7367 kDefaultFramerate);
7368
7369 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7370 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7371 timestamp_ms += 1000 / kDefaultFramerate;
7372 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7373 WaitForEncodedFrame(timestamp_ms);
7374 }
7375 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7376 kDefaultFramerate);
7377 // Capture larger frame to trigger a reconfigure.
7378 codec_height_ *= 2;
7379 codec_width_ *= 2;
7380 timestamp_ms += 1000 / kDefaultFramerate;
7381 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7382 WaitForEncodedFrame(timestamp_ms);
7383
7384 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7385 auto current_rate_settings =
7386 fake_encoder_.GetAndResetLastRateControlSettings();
7387 // Ensure we have actually reconfigured twice
7388 // The rate settings should have been set again even though
7389 // they haven't changed.
7390 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007391 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007392
7393 video_stream_encoder_->Stop();
7394}
7395
philipeld9cc8c02019-09-16 14:53:40 +02007396struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007397 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7398 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7399 MOCK_METHOD(void,
7400 RequestEncoderSwitch,
7401 (const webrtc::SdpVideoFormat& format),
7402 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007403};
7404
philipel9b058032020-02-10 11:30:00 +01007405TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7406 constexpr int kDontCare = 100;
7407 StrictMock<MockEncoderSelector> encoder_selector;
7408 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7409 &fake_encoder_, &encoder_selector);
7410 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7411
7412 // Reset encoder for new configuration to take effect.
7413 ConfigureEncoder(video_encoder_config_.Copy());
7414
7415 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7416
7417 video_source_.IncomingCapturedFrame(
7418 CreateFrame(kDontCare, kDontCare, kDontCare));
7419 video_stream_encoder_->Stop();
7420
7421 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7422 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007423 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7424 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007425 video_stream_encoder_.reset();
7426}
7427
7428TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7429 constexpr int kDontCare = 100;
7430
7431 NiceMock<MockEncoderSelector> encoder_selector;
7432 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7433 video_send_config_.encoder_settings.encoder_switch_request_callback =
7434 &switch_callback;
7435 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7436 &fake_encoder_, &encoder_selector);
7437 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7438
7439 // Reset encoder for new configuration to take effect.
7440 ConfigureEncoder(video_encoder_config_.Copy());
7441
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007442 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007443 .WillByDefault(Return(SdpVideoFormat("AV1")));
7444 EXPECT_CALL(switch_callback,
7445 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7446 Field(&SdpVideoFormat::name, "AV1"))));
7447
Henrik Boström381d1092020-05-12 18:49:07 +02007448 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007449 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7450 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7451 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007452 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007453 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007454 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007455 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007456
7457 video_stream_encoder_->Stop();
7458}
7459
7460TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7461 constexpr int kSufficientBitrateToNotDrop = 1000;
7462 constexpr int kDontCare = 100;
7463
7464 NiceMock<MockVideoEncoder> video_encoder;
7465 NiceMock<MockEncoderSelector> encoder_selector;
7466 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7467 video_send_config_.encoder_settings.encoder_switch_request_callback =
7468 &switch_callback;
7469 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7470 &video_encoder, &encoder_selector);
7471 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7472
7473 // Reset encoder for new configuration to take effect.
7474 ConfigureEncoder(video_encoder_config_.Copy());
7475
7476 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7477 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7478 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007479 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007480 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7481 /*stable_target_bitrate=*/
7482 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7483 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007484 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007485 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007486 /*cwnd_reduce_ratio=*/0);
7487
7488 ON_CALL(video_encoder, Encode(_, _))
7489 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7490 ON_CALL(encoder_selector, OnEncoderBroken())
7491 .WillByDefault(Return(SdpVideoFormat("AV2")));
7492
7493 rtc::Event encode_attempted;
7494 EXPECT_CALL(switch_callback,
7495 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7496 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7497 EXPECT_EQ(format.name, "AV2");
7498 encode_attempted.Set();
7499 });
7500
7501 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7502 encode_attempted.Wait(3000);
7503
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007504 AdvanceTime(TimeDelta::Millis(0));
7505
philipel9b058032020-02-10 11:30:00 +01007506 video_stream_encoder_->Stop();
7507
7508 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7509 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007510 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7511 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007512 video_stream_encoder_.reset();
7513}
7514
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007515TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007516 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007517 const int kFrameWidth = 320;
7518 const int kFrameHeight = 180;
7519
7520 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007521 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007523 /*target_bitrate=*/rate,
7524 /*stable_target_bitrate=*/rate,
7525 /*link_allocation=*/rate,
7526 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007527 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007528 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007529
7530 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007531 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007532 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7533 frame.set_rotation(kVideoRotation_270);
7534 video_source_.IncomingCapturedFrame(frame);
7535 WaitForEncodedFrame(timestamp_ms);
7536 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7537
7538 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007539 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007541 /*target_bitrate=*/new_stable_rate,
7542 /*stable_target_bitrate=*/new_stable_rate,
7543 /*link_allocation=*/rate,
7544 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007545 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007546 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007547 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7548 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7549 video_stream_encoder_->Stop();
7550}
7551
7552TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007553 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007554 const int kFrameWidth = 320;
7555 const int kFrameHeight = 180;
7556
7557 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007558 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007560 /*target_bitrate=*/rate,
7561 /*stable_target_bitrate=*/rate,
7562 /*link_allocation=*/rate,
7563 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007564 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007565 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007566
7567 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007568 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007569 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7570 frame.set_rotation(kVideoRotation_270);
7571 video_source_.IncomingCapturedFrame(frame);
7572 WaitForEncodedFrame(timestamp_ms);
7573 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7574
7575 // Set a higher target rate without changing the link_allocation. Should not
7576 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007577 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007579 /*target_bitrate=*/rate,
7580 /*stable_target_bitrate=*/new_stable_rate,
7581 /*link_allocation=*/rate,
7582 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007583 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007584 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007585 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7586 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7587 video_stream_encoder_->Stop();
7588}
7589
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007590TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7591 test::ScopedFieldTrials field_trials(
7592 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7593 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7594 const int kFramerateFps = 30;
7595 const int kWidth = 1920;
7596 const int kHeight = 1080;
7597 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7598 // Works on screenshare mode.
7599 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7600 // We rely on the automatic resolution adaptation, but we handle framerate
7601 // adaptation manually by mocking the stats proxy.
7602 video_source_.set_adaptation_enabled(true);
7603
7604 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007607 video_stream_encoder_->SetSource(&video_source_,
7608 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007609 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007610
7611 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7612 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7613
7614 // Pass enough frames with the full update to trigger animation detection.
7615 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007616 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007617 frame.set_ntp_time_ms(timestamp_ms);
7618 frame.set_timestamp_us(timestamp_ms * 1000);
7619 video_source_.IncomingCapturedFrame(frame);
7620 WaitForEncodedFrame(timestamp_ms);
7621 }
7622
7623 // Resolution should be limited.
7624 rtc::VideoSinkWants expected;
7625 expected.max_framerate_fps = kFramerateFps;
7626 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007627 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007628
7629 // Pass one frame with no known update.
7630 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007631 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007632 frame.set_ntp_time_ms(timestamp_ms);
7633 frame.set_timestamp_us(timestamp_ms * 1000);
7634 frame.clear_update_rect();
7635
7636 video_source_.IncomingCapturedFrame(frame);
7637 WaitForEncodedFrame(timestamp_ms);
7638
7639 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007640 EXPECT_THAT(video_source_.sink_wants(),
7641 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007642
7643 video_stream_encoder_->Stop();
7644}
7645
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007646TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7647 const int kWidth = 720; // 540p adapted down.
7648 const int kHeight = 405;
7649 const int kNumFrames = 3;
7650 // Works on screenshare mode.
7651 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7652 /*num_spatial_layers=*/2, /*screenshare=*/true);
7653
7654 video_source_.set_adaptation_enabled(true);
7655
7656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007657 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007658
7659 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7660
7661 // Pass enough frames with the full update to trigger animation detection.
7662 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007663 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007664 frame.set_ntp_time_ms(timestamp_ms);
7665 frame.set_timestamp_us(timestamp_ms * 1000);
7666 video_source_.IncomingCapturedFrame(frame);
7667 WaitForEncodedFrame(timestamp_ms);
7668 }
7669
7670 video_stream_encoder_->Stop();
7671}
7672
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007673TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7674 const float downscale_factors[] = {4.0, 2.0, 1.0};
7675 const int number_layers =
7676 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7677 VideoEncoderConfig config;
7678 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7679 for (int i = 0; i < number_layers; ++i) {
7680 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7681 config.simulcast_layers[i].active = true;
7682 }
7683 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007684 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007685 "VP8", /*max qp*/ 56, /*screencast*/ false,
7686 /*screenshare enabled*/ false);
7687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007688 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7689 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007690
7691 // First initialization.
7692 // Encoder should be initialized. Next frame should be key frame.
7693 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7694 sink_.SetNumExpectedLayers(number_layers);
7695 int64_t timestamp_ms = kFrameIntervalMs;
7696 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7697 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007698 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007699 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7700 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7701 VideoFrameType::kVideoFrameKey,
7702 VideoFrameType::kVideoFrameKey}));
7703
7704 // Disable top layer.
7705 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7706 config.simulcast_layers[number_layers - 1].active = false;
7707 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7708 sink_.SetNumExpectedLayers(number_layers - 1);
7709 timestamp_ms += kFrameIntervalMs;
7710 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7711 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007712 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007713 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7714 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7715 VideoFrameType::kVideoFrameDelta,
7716 VideoFrameType::kVideoFrameDelta}));
7717
7718 // Re-enable top layer.
7719 // Encoder should be re-initialized. Next frame should be key frame.
7720 config.simulcast_layers[number_layers - 1].active = true;
7721 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7722 sink_.SetNumExpectedLayers(number_layers);
7723 timestamp_ms += kFrameIntervalMs;
7724 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7725 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007726 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007727 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7728 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7729 VideoFrameType::kVideoFrameKey,
7730 VideoFrameType::kVideoFrameKey}));
7731
7732 // Top layer max rate change.
7733 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7734 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7735 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7736 sink_.SetNumExpectedLayers(number_layers);
7737 timestamp_ms += kFrameIntervalMs;
7738 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7739 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007740 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007741 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7742 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7743 VideoFrameType::kVideoFrameDelta,
7744 VideoFrameType::kVideoFrameDelta}));
7745
7746 // Top layer resolution change.
7747 // Encoder should be re-initialized. Next frame should be key frame.
7748 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7749 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7750 sink_.SetNumExpectedLayers(number_layers);
7751 timestamp_ms += kFrameIntervalMs;
7752 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7753 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007754 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007755 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7756 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7757 VideoFrameType::kVideoFrameKey,
7758 VideoFrameType::kVideoFrameKey}));
7759 video_stream_encoder_->Stop();
7760}
7761
Henrik Boström1124ed12021-02-25 10:30:39 +01007762TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7763 const int kFrameWidth = 1280;
7764 const int kFrameHeight = 720;
7765
7766 SetUp();
7767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007768 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007769
7770 // Capturing a frame should reconfigure the encoder and expose the encoder
7771 // resolution, which is the same as the input frame.
7772 int64_t timestamp_ms = kFrameIntervalMs;
7773 video_source_.IncomingCapturedFrame(
7774 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7775 WaitForEncodedFrame(timestamp_ms);
7776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7777 EXPECT_THAT(video_source_.sink_wants().resolutions,
7778 ::testing::ElementsAreArray(
7779 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7780
7781 video_stream_encoder_->Stop();
7782}
7783
7784TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7785 // Pick downscale factors such that we never encode at full resolution - this
7786 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007787 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007788 // encoder should not ask for the frame resolution. This allows video frames
7789 // to have the appearence of one resolution but optimize its internal buffers
7790 // for what is actually encoded.
7791 const size_t kNumSimulcastLayers = 3u;
7792 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7793 const int kFrameWidth = 1280;
7794 const int kFrameHeight = 720;
7795 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7796 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7797 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7798 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7799 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7800 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7801
7802 VideoEncoderConfig config;
7803 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7804 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7805 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7806 config.simulcast_layers[i].active = true;
7807 }
7808 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007809 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007810 "VP8", /*max qp*/ 56, /*screencast*/ false,
7811 /*screenshare enabled*/ false);
7812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007813 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7814 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007815
7816 // Capture a frame with all layers active.
7817 int64_t timestamp_ms = kFrameIntervalMs;
7818 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7819 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7820 video_source_.IncomingCapturedFrame(
7821 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7822 WaitForEncodedFrame(timestamp_ms);
7823 // Expect encoded resolutions to match the expected simulcast layers.
7824 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7825 EXPECT_THAT(
7826 video_source_.sink_wants().resolutions,
7827 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7828
7829 // Capture a frame with one of the layers inactive.
7830 timestamp_ms += kFrameIntervalMs;
7831 config.simulcast_layers[2].active = false;
7832 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7833 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7834 video_source_.IncomingCapturedFrame(
7835 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7836 WaitForEncodedFrame(timestamp_ms);
7837
7838 // Expect encoded resolutions to match the expected simulcast layers.
7839 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7840 EXPECT_THAT(video_source_.sink_wants().resolutions,
7841 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7842
7843 // Capture a frame with all but one layer turned off.
7844 timestamp_ms += kFrameIntervalMs;
7845 config.simulcast_layers[1].active = false;
7846 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7847 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7848 video_source_.IncomingCapturedFrame(
7849 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7850 WaitForEncodedFrame(timestamp_ms);
7851
7852 // Expect encoded resolutions to match the expected simulcast layers.
7853 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7854 EXPECT_THAT(video_source_.sink_wants().resolutions,
7855 ::testing::ElementsAreArray({kLayer0Size}));
7856
7857 video_stream_encoder_->Stop();
7858}
7859
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007860TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007861 ResetEncoder("VP8", 1, 1, 1, false);
7862
Niels Möller8b692902021-06-14 12:04:57 +02007863 // Force encoder reconfig.
7864 video_source_.IncomingCapturedFrame(
7865 CreateFrame(1, codec_width_, codec_height_));
7866 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7867
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007868 // Set QP on encoded frame and pass the frame to encode complete callback.
7869 // Since QP is present QP parsing won't be triggered and the original value
7870 // should be kept.
7871 EncodedImage encoded_image;
7872 encoded_image.qp_ = 123;
7873 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7874 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7875 CodecSpecificInfo codec_info;
7876 codec_info.codecType = kVideoCodecVP8;
7877 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7878 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7879 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7880 video_stream_encoder_->Stop();
7881}
7882
7883TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007884 ResetEncoder("VP8", 1, 1, 1, false);
7885
Niels Möller8b692902021-06-14 12:04:57 +02007886 // Force encoder reconfig.
7887 video_source_.IncomingCapturedFrame(
7888 CreateFrame(1, codec_width_, codec_height_));
7889 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7890
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007891 // Pass an encoded frame without QP to encode complete callback. QP should be
7892 // parsed and set.
7893 EncodedImage encoded_image;
7894 encoded_image.qp_ = -1;
7895 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7896 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7897 CodecSpecificInfo codec_info;
7898 codec_info.codecType = kVideoCodecVP8;
7899 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7900 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7901 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7902 video_stream_encoder_->Stop();
7903}
7904
7905TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7906 webrtc::test::ScopedFieldTrials field_trials(
7907 "WebRTC-QpParsingKillSwitch/Enabled/");
7908
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007909 ResetEncoder("VP8", 1, 1, 1, false);
7910
Niels Möller8b692902021-06-14 12:04:57 +02007911 // Force encoder reconfig.
7912 video_source_.IncomingCapturedFrame(
7913 CreateFrame(1, codec_width_, codec_height_));
7914 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7915
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007916 EncodedImage encoded_image;
7917 encoded_image.qp_ = -1;
7918 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7919 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7920 CodecSpecificInfo codec_info;
7921 codec_info.codecType = kVideoCodecVP8;
7922 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7923 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7924 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7925 video_stream_encoder_->Stop();
7926}
7927
Sergey Silkind19e3b92021-03-16 10:05:30 +00007928TEST_F(VideoStreamEncoderTest,
7929 QualityScalingNotAllowed_QualityScalingDisabled) {
7930 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7931
7932 // Disable scaling settings in encoder info.
7933 fake_encoder_.SetQualityScaling(false);
7934 // Disable quality scaling in encoder config.
7935 video_encoder_config.is_quality_scaling_allowed = false;
7936 ConfigureEncoder(std::move(video_encoder_config));
7937
7938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007939 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007940
7941 test::FrameForwarder source;
7942 video_stream_encoder_->SetSource(
7943 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7944 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7945 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7946
7947 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7948 WaitForEncodedFrame(1);
7949 video_stream_encoder_->TriggerQualityLow();
7950 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7951
7952 video_stream_encoder_->Stop();
7953}
7954
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007955TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
7956 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7957
7958 // Disable scaling settings in encoder info.
7959 fake_encoder_.SetQualityScaling(false);
7960 // Set QP trusted in encoder info.
7961 fake_encoder_.SetIsQpTrusted(true);
7962 // Enable quality scaling in encoder config.
7963 video_encoder_config.is_quality_scaling_allowed = false;
7964 ConfigureEncoder(std::move(video_encoder_config));
7965
7966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007967 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007968
7969 test::FrameForwarder source;
7970 video_stream_encoder_->SetSource(
7971 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7972 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7973 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7974
7975 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7976 WaitForEncodedFrame(1);
7977 video_stream_encoder_->TriggerQualityLow();
7978 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7979
7980 video_stream_encoder_->Stop();
7981}
7982
Shuhai Pengf2707702021-09-29 17:19:44 +08007983TEST_F(VideoStreamEncoderTest,
7984 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
7985 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7986
7987 // Disable scaling settings in encoder info.
7988 fake_encoder_.SetQualityScaling(false);
7989 // Set QP trusted in encoder info.
7990 fake_encoder_.SetIsQpTrusted(true);
7991 // Enable quality scaling in encoder config.
7992 video_encoder_config.is_quality_scaling_allowed = false;
7993 ConfigureEncoder(std::move(video_encoder_config));
7994
7995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007996 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08007997
7998 test::FrameForwarder source;
7999 video_stream_encoder_->SetSource(
8000 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8001 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8002 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8003
8004 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8005 WaitForEncodedFrame(1);
8006 video_stream_encoder_->TriggerQualityLow();
8007 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8008
8009 video_stream_encoder_->Stop();
8010}
8011
8012TEST_F(VideoStreamEncoderTest,
8013 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8014 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8015
8016 // Disable scaling settings in encoder info.
8017 fake_encoder_.SetQualityScaling(false);
8018 // Set QP trusted in encoder info.
8019 fake_encoder_.SetIsQpTrusted(false);
8020 // Enable quality scaling in encoder config.
8021 video_encoder_config.is_quality_scaling_allowed = false;
8022 ConfigureEncoder(std::move(video_encoder_config));
8023
8024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008025 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008026
8027 test::FrameForwarder source;
8028 video_stream_encoder_->SetSource(
8029 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8030 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8031 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8032
8033 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8034 WaitForEncodedFrame(1);
8035 video_stream_encoder_->TriggerQualityLow();
8036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8037
8038 video_stream_encoder_->Stop();
8039}
8040
8041TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8042 // Set QP trusted in encoder info.
8043 fake_encoder_.SetIsQpTrusted(false);
8044
8045 const int MinEncBitrateKbps = 30;
8046 const int MaxEncBitrateKbps = 100;
8047 const int MinStartBitrateKbp = 50;
8048 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8049 /*frame_size_pixels=*/codec_width_ * codec_height_,
8050 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8051 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8052 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8053
8054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008055 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008056
8057 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8058
8059 VideoEncoderConfig video_encoder_config;
8060 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8061 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8062 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8063 MinEncBitrateKbps * 1000;
8064 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8065 kMaxPayloadLength);
8066
8067 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8068 WaitForEncodedFrame(1);
8069 EXPECT_EQ(
8070 MaxEncBitrateKbps,
8071 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8072 EXPECT_EQ(
8073 MinEncBitrateKbps,
8074 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8075
8076 video_stream_encoder_->Stop();
8077}
8078
8079TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8080 // Set QP trusted in encoder info.
8081 fake_encoder_.SetIsQpTrusted(false);
8082
8083 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8084 EncoderInfoSettings::
8085 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8086 codec_width_ * codec_height_,
8087 EncoderInfoSettings::
8088 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8089 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8090
8091 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8092 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8093 const int TargetEncBitrate = MaxEncBitrate;
8094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8095 DataRate::BitsPerSec(TargetEncBitrate),
8096 DataRate::BitsPerSec(TargetEncBitrate),
8097 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8098
8099 VideoEncoderConfig video_encoder_config;
8100 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8101 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8102 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8103 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8104 kMaxPayloadLength);
8105
8106 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8107 WaitForEncodedFrame(1);
8108 EXPECT_EQ(
8109 MaxEncBitrate / 1000,
8110 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8111 EXPECT_EQ(
8112 MinEncBitrate / 1000,
8113 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8114
8115 video_stream_encoder_->Stop();
8116}
8117
Sergey Silkind19e3b92021-03-16 10:05:30 +00008118#if !defined(WEBRTC_IOS)
8119// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8120// disabled by default on iOS.
8121TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8122 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8123
8124 // Disable scaling settings in encoder info.
8125 fake_encoder_.SetQualityScaling(false);
8126 // Enable quality scaling in encoder config.
8127 video_encoder_config.is_quality_scaling_allowed = true;
8128 ConfigureEncoder(std::move(video_encoder_config));
8129
8130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008131 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008132
8133 test::FrameForwarder source;
8134 video_stream_encoder_->SetSource(
8135 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8136 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8137 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8138
8139 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8140 WaitForEncodedFrame(1);
8141 video_stream_encoder_->TriggerQualityLow();
8142 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8143
8144 video_stream_encoder_->Stop();
8145}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008146
8147TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8148 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8149
8150 // Disable scaling settings in encoder info.
8151 fake_encoder_.SetQualityScaling(false);
8152 // Set QP trusted in encoder info.
8153 fake_encoder_.SetIsQpTrusted(true);
8154 // Enable quality scaling in encoder config.
8155 video_encoder_config.is_quality_scaling_allowed = true;
8156 ConfigureEncoder(std::move(video_encoder_config));
8157
8158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008159 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008160
8161 test::FrameForwarder source;
8162 video_stream_encoder_->SetSource(
8163 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8164 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8165 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8166
8167 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8168 WaitForEncodedFrame(1);
8169 video_stream_encoder_->TriggerQualityLow();
8170 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8171
8172 video_stream_encoder_->Stop();
8173}
Shuhai Pengf2707702021-09-29 17:19:44 +08008174
8175TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8176 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8177
8178 // Disable scaling settings in encoder info.
8179 fake_encoder_.SetQualityScaling(false);
8180 // Set QP not trusted in encoder info.
8181 fake_encoder_.SetIsQpTrusted(false);
8182 // Enable quality scaling in encoder config.
8183 video_encoder_config.is_quality_scaling_allowed = true;
8184 ConfigureEncoder(std::move(video_encoder_config));
8185
8186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008187 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008188
8189 test::FrameForwarder source;
8190 video_stream_encoder_->SetSource(
8191 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8192 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8193 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8194
8195 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8196 WaitForEncodedFrame(1);
8197 video_stream_encoder_->TriggerQualityLow();
8198 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8199 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8200 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8201
8202 video_stream_encoder_->Stop();
8203}
8204
8205TEST_F(VideoStreamEncoderTest,
8206 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8207 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8208
8209 // Disable scaling settings in encoder info.
8210 fake_encoder_.SetQualityScaling(false);
8211 // Set QP trusted in encoder info.
8212 fake_encoder_.SetIsQpTrusted(true);
8213 // Enable quality scaling in encoder config.
8214 video_encoder_config.is_quality_scaling_allowed = true;
8215 ConfigureEncoder(std::move(video_encoder_config));
8216
8217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008218 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008219
8220 test::FrameForwarder source;
8221 video_stream_encoder_->SetSource(
8222 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8223 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8224 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8225
8226 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8227 WaitForEncodedFrame(1);
8228 video_stream_encoder_->TriggerQualityLow();
8229 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8230 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8231
8232 video_stream_encoder_->Stop();
8233}
8234
8235TEST_F(VideoStreamEncoderTest,
8236 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8237 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8238
8239 // Disable scaling settings in encoder info.
8240 fake_encoder_.SetQualityScaling(false);
8241 // Set QP trusted in encoder info.
8242 fake_encoder_.SetIsQpTrusted(false);
8243 // Enable quality scaling in encoder config.
8244 video_encoder_config.is_quality_scaling_allowed = true;
8245 ConfigureEncoder(std::move(video_encoder_config));
8246
8247 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008248 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008249
8250 test::FrameForwarder source;
8251 video_stream_encoder_->SetSource(
8252 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8253 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8254 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8255
8256 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8257 WaitForEncodedFrame(1);
8258 video_stream_encoder_->TriggerQualityLow();
8259 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8260
8261 video_stream_encoder_->Stop();
8262}
8263
Sergey Silkind19e3b92021-03-16 10:05:30 +00008264#endif
8265
Henrik Boström56db9ff2021-03-24 09:06:45 +01008266// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8267class VideoStreamEncoderWithRealEncoderTest
8268 : public VideoStreamEncoderTest,
8269 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8270 public:
8271 VideoStreamEncoderWithRealEncoderTest()
8272 : VideoStreamEncoderTest(),
8273 codec_type_(std::get<0>(GetParam())),
8274 allow_i420_conversion_(std::get<1>(GetParam())) {}
8275
8276 void SetUp() override {
8277 VideoStreamEncoderTest::SetUp();
8278 std::unique_ptr<VideoEncoder> encoder;
8279 switch (codec_type_) {
8280 case kVideoCodecVP8:
8281 encoder = VP8Encoder::Create();
8282 break;
8283 case kVideoCodecVP9:
8284 encoder = VP9Encoder::Create();
8285 break;
8286 case kVideoCodecAV1:
8287 encoder = CreateLibaomAv1Encoder();
8288 break;
8289 case kVideoCodecH264:
8290 encoder =
8291 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8292 break;
8293 case kVideoCodecMultiplex:
8294 mock_encoder_factory_for_multiplex_ =
8295 std::make_unique<MockVideoEncoderFactory>();
8296 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8297 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8298 .WillRepeatedly([] { return VP8Encoder::Create(); });
8299 encoder = std::make_unique<MultiplexEncoderAdapter>(
8300 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8301 false);
8302 break;
8303 default:
8304 RTC_NOTREACHED();
8305 }
8306 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8307 }
8308
8309 void TearDown() override {
8310 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008311 // Ensure `video_stream_encoder_` is destroyed before
8312 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008313 video_stream_encoder_.reset();
8314 VideoStreamEncoderTest::TearDown();
8315 }
8316
8317 protected:
8318 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8319 std::unique_ptr<VideoEncoder> encoder) {
8320 // Configure VSE to use the encoder.
8321 encoder_ = std::move(encoder);
8322 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8323 encoder_.get(), &encoder_selector_);
8324 video_send_config_.encoder_settings.encoder_factory =
8325 encoder_proxy_factory_.get();
8326 VideoEncoderConfig video_encoder_config;
8327 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8328 video_encoder_config_ = video_encoder_config.Copy();
8329 ConfigureEncoder(video_encoder_config_.Copy());
8330
8331 // Set bitrate to ensure frame is not dropped.
8332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008333 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008334 }
8335
8336 const VideoCodecType codec_type_;
8337 const bool allow_i420_conversion_;
8338 NiceMock<MockEncoderSelector> encoder_selector_;
8339 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8340 std::unique_ptr<VideoEncoder> encoder_;
8341 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8342};
8343
8344TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8345 auto native_i420_frame = test::CreateMappableNativeFrame(
8346 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8347 video_source_.IncomingCapturedFrame(native_i420_frame);
8348 WaitForEncodedFrame(codec_width_, codec_height_);
8349
8350 auto mappable_native_buffer =
8351 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8352 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8353 mappable_native_buffer->GetMappedFramedBuffers();
8354 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8355 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8356 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8357 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8358}
8359
8360TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8361 auto native_nv12_frame = test::CreateMappableNativeFrame(
8362 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8363 video_source_.IncomingCapturedFrame(native_nv12_frame);
8364 WaitForEncodedFrame(codec_width_, codec_height_);
8365
8366 auto mappable_native_buffer =
8367 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8368 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8369 mappable_native_buffer->GetMappedFramedBuffers();
8370 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8371 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8372 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8373 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8374
8375 if (!allow_i420_conversion_) {
8376 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8377 }
8378}
8379
Erik Språng7444b192021-06-02 14:02:13 +02008380TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8381 if (codec_type_ == kVideoCodecMultiplex) {
8382 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8383 return;
8384 }
8385
8386 const size_t kNumSpatialLayers = 3u;
8387 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8388 const int kFrameWidth = 1280;
8389 const int kFrameHeight = 720;
8390 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8391 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8392 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8393 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8394 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8395 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8396
8397 VideoEncoderConfig config;
8398 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8399 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008400 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008401 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8402 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8403 vp9_settings.numberOfTemporalLayers = 3;
8404 vp9_settings.automaticResizeOn = false;
8405 config.encoder_specific_settings =
8406 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8407 vp9_settings);
8408 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8409 /*fps=*/30.0,
8410 /*first_active_layer=*/0,
8411 /*num_spatial_layers=*/3,
8412 /*num_temporal_layers=*/3,
8413 /*is_screenshare=*/false);
8414 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8415 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008416 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008417 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8418 /*fps=*/30.0,
8419 /*first_active_layer=*/0,
8420 /*num_spatial_layers=*/3,
8421 /*num_temporal_layers=*/3,
8422 /*is_screenshare=*/false);
8423 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8424 } else {
8425 // Simulcast for VP8/H264.
8426 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8427 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8428 config.simulcast_layers[i].scale_resolution_down_by =
8429 kDownscaleFactors[i];
8430 config.simulcast_layers[i].active = true;
8431 }
8432 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8433 // Turn off frame dropping to prevent flakiness.
8434 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8435 h264_settings.frameDroppingOn = false;
8436 config.encoder_specific_settings = rtc::make_ref_counted<
8437 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8438 }
8439 }
8440
8441 auto set_layer_active = [&](int layer_idx, bool active) {
8442 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8443 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8444 config.spatial_layers[layer_idx].active = active;
8445 } else {
8446 config.simulcast_layers[layer_idx].active = active;
8447 }
8448 };
8449
8450 config.video_stream_factory =
8451 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8452 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8453 /*screencast*/ false,
8454 /*screenshare enabled*/ false);
8455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008456 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8457 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008458
8459 // Capture a frame with all layers active.
8460 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8461 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8462 int64_t timestamp_ms = kFrameIntervalMs;
8463 video_source_.IncomingCapturedFrame(
8464 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8465
8466 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8467 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8468
8469 // Capture a frame with one of the layers inactive.
8470 set_layer_active(2, false);
8471 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8472 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8473 timestamp_ms += kFrameIntervalMs;
8474 video_source_.IncomingCapturedFrame(
8475 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8476 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8477
8478 // New target bitrates signaled based on lower resolution.
8479 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8481 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8482 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8483
8484 // Re-enable the top layer.
8485 set_layer_active(2, true);
8486 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8487 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8488 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8489
8490 // Bitrate target adjusted back up to enable HD layer...
8491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8492 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8493 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8494 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8495
8496 // ...then add a new frame.
8497 timestamp_ms += kFrameIntervalMs;
8498 video_source_.IncomingCapturedFrame(
8499 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8500 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8501 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8502
8503 video_stream_encoder_->Stop();
8504}
8505
Henrik Boström56db9ff2021-03-24 09:06:45 +01008506std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8507 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8508 VideoCodecType codec_type = std::get<0>(info.param);
8509 bool allow_i420_conversion = std::get<1>(info.param);
8510 std::string str;
8511 switch (codec_type) {
8512 case kVideoCodecGeneric:
8513 str = "Generic";
8514 break;
8515 case kVideoCodecVP8:
8516 str = "VP8";
8517 break;
8518 case kVideoCodecVP9:
8519 str = "VP9";
8520 break;
8521 case kVideoCodecAV1:
8522 str = "AV1";
8523 break;
8524 case kVideoCodecH264:
8525 str = "H264";
8526 break;
8527 case kVideoCodecMultiplex:
8528 str = "Multiplex";
8529 break;
8530 default:
8531 RTC_NOTREACHED();
8532 }
8533 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8534 return str;
8535}
8536
8537constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8538 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8539constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8540 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8541constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8542 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8543constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8544 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8545#if defined(WEBRTC_USE_H264)
8546constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8547 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8548
8549// The windows compiler does not tolerate #if statements inside the
8550// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8551// and without H264).
8552INSTANTIATE_TEST_SUITE_P(
8553 All,
8554 VideoStreamEncoderWithRealEncoderTest,
8555 ::testing::Values(kVP8DisallowConversion,
8556 kVP9DisallowConversion,
8557 kAV1AllowConversion,
8558 kMultiplexDisallowConversion,
8559 kH264AllowConversion),
8560 TestParametersVideoCodecAndAllowI420ConversionToString);
8561#else
8562INSTANTIATE_TEST_SUITE_P(
8563 All,
8564 VideoStreamEncoderWithRealEncoderTest,
8565 ::testing::Values(kVP8DisallowConversion,
8566 kVP9DisallowConversion,
8567 kAV1AllowConversion,
8568 kMultiplexDisallowConversion),
8569 TestParametersVideoCodecAndAllowI420ConversionToString);
8570#endif
8571
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008572class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8573 protected:
8574 void RunTest(const std::vector<VideoStream>& configs,
8575 const int expected_num_init_encode) {
8576 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008577 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008578 InsertFrameAndWaitForEncoded();
8579 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8580 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008581 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8582 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008583
8584 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8585 ConfigureEncoder(configs[1]);
8586 InsertFrameAndWaitForEncoded();
8587 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8588 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008589 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008590 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008591 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008592
8593 video_stream_encoder_->Stop();
8594 }
8595
8596 void ConfigureEncoder(const VideoStream& stream) {
8597 VideoEncoderConfig config;
8598 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8599 config.max_bitrate_bps = stream.max_bitrate_bps;
8600 config.simulcast_layers[0] = stream;
8601 config.video_stream_factory =
8602 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8603 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8604 /*conference_mode=*/false);
8605 video_stream_encoder_->ConfigureEncoder(std::move(config),
8606 kMaxPayloadLength);
8607 }
8608
8609 void OnBitrateUpdated(DataRate bitrate) {
8610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8611 bitrate, bitrate, bitrate, 0, 0, 0);
8612 }
8613
8614 void InsertFrameAndWaitForEncoded() {
8615 timestamp_ms_ += kFrameIntervalMs;
8616 video_source_.IncomingCapturedFrame(
8617 CreateFrame(timestamp_ms_, kWidth, kHeight));
8618 sink_.WaitForEncodedFrame(timestamp_ms_);
8619 }
8620
8621 void ExpectEqual(const VideoCodec& actual,
8622 const VideoStream& expected) const {
8623 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8624 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8625 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8626 static_cast<unsigned int>(expected.min_bitrate_bps));
8627 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8628 static_cast<unsigned int>(expected.max_bitrate_bps));
8629 EXPECT_EQ(actual.simulcastStream[0].width,
8630 kWidth / expected.scale_resolution_down_by);
8631 EXPECT_EQ(actual.simulcastStream[0].height,
8632 kHeight / expected.scale_resolution_down_by);
8633 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8634 expected.num_temporal_layers);
8635 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8636 }
8637
8638 VideoStream DefaultConfig() const {
8639 VideoStream stream;
8640 stream.max_framerate = 25;
8641 stream.min_bitrate_bps = 35000;
8642 stream.max_bitrate_bps = 900000;
8643 stream.scale_resolution_down_by = 1.0;
8644 stream.num_temporal_layers = 1;
8645 stream.bitrate_priority = 1.0;
8646 stream.scalability_mode = "";
8647 return stream;
8648 }
8649
8650 const int kWidth = 640;
8651 const int kHeight = 360;
8652 int64_t timestamp_ms_ = 0;
8653};
8654
8655TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8656 VideoStream config1 = DefaultConfig();
8657 VideoStream config2 = config1;
8658 config2.max_framerate++;
8659
8660 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8661}
8662
8663TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8664 VideoStream config1 = DefaultConfig();
8665 VideoStream config2 = config1;
8666 config2.min_bitrate_bps += 10000;
8667
8668 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8669}
8670
8671TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8672 VideoStream config1 = DefaultConfig();
8673 VideoStream config2 = config1;
8674 config2.max_bitrate_bps += 100000;
8675
8676 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8677}
8678
8679TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8680 VideoStream config1 = DefaultConfig();
8681 VideoStream config2 = config1;
8682 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8683
8684 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8685}
8686
8687TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8688 VideoStream config1 = DefaultConfig();
8689 VideoStream config2 = config1;
8690 config2.scale_resolution_down_by *= 2;
8691
8692 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8693}
8694
8695TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8696 VideoStream config1 = DefaultConfig();
8697 VideoStream config2 = config1;
8698 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8699
8700 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8701}
8702
8703TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8704 VideoStream config1 = DefaultConfig();
8705 VideoStream config2 = config1;
8706 config2.scalability_mode = "L1T2";
8707
8708 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8709}
8710
perkj26091b12016-09-01 01:17:40 -07008711} // namespace webrtc