blob: b41b4b0d1da5a70d38f1aeffb6e45549e11e67b6 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020045#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020046#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010047#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020048#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010049#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020050#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020053#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010054#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010062#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020063#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020064#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070066
67namespace webrtc {
68
sprang57c2fff2017-01-16 06:24:02 -080069using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020071using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020072using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010081using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080083
perkj803d97f2016-11-01 11:45:46 -070084namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080091const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020092const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
93const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
94const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
95const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -080096const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070097const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020099const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800104
Asa Persson606d3cb2021-10-04 10:07:11 +0200105uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 11:45:46 -0700115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 09:06:45 +0100129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150 }
Noah Richards51db4212019-06-12 06:59:12 -0700151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 09:15:13 +0200163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 10:27:48 +0100194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 09:04:13 +0200215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 18:49:07 +0200217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219 private:
Markus Handella3765182020-07-08 13:13:32 +0200220 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100223};
224
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200227 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 08:27:51 -0700343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700344 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100358 task_queue_factory,
Markus Handell2b10c472021-10-28 15:29:42 +0200359 TaskQueueBase::Current(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100360 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200361 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200362 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200363 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200364 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200365 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200366 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200367 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200368 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100369 }
perkj803d97f2016-11-01 11:45:46 -0700370
Henrik Boström381d1092020-05-12 18:49:07 +0200371 void SetSourceAndWaitForRestrictionsUpdated(
372 rtc::VideoSourceInterface<VideoFrame>* source,
373 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200374 FakeVideoSourceRestrictionsListener listener;
375 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200376 SetSource(source, degradation_preference);
377 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200378 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200379 }
380
381 void SetSourceAndWaitForFramerateUpdated(
382 rtc::VideoSourceInterface<VideoFrame>* source,
383 const DegradationPreference& degradation_preference) {
384 overuse_detector_proxy_->framerate_updated_event()->Reset();
385 SetSource(source, degradation_preference);
386 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
387 }
388
389 void OnBitrateUpdatedAndWaitForManagedResources(
390 DataRate target_bitrate,
391 DataRate stable_target_bitrate,
392 DataRate link_allocation,
393 uint8_t fraction_lost,
394 int64_t round_trip_time_ms,
395 double cwnd_reduce_ratio) {
396 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
397 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
398 // Bitrate is updated on the encoder queue.
399 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200400 }
401
kthelgason2fc52542017-03-03 00:24:41 -0800402 // This is used as a synchronisation mechanism, to make sure that the
403 // encoder queue is not blocked before we start sending it frames.
404 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100405 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200406 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800407 ASSERT_TRUE(event.Wait(5000));
408 }
409
Henrik Boström91aa7322020-04-28 12:24:33 +0200410 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200411 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200412 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200413 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200414 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200415 event.Set();
416 });
417 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200418 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200419 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200420
Henrik Boström91aa7322020-04-28 12:24:33 +0200421 void TriggerCpuUnderuse() {
422 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200423 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200424 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200425 event.Set();
426 });
427 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200428 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200429 }
kthelgason876222f2016-11-29 01:44:11 -0800430
Henrik Boström91aa7322020-04-28 12:24:33 +0200431 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200432 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200433 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200434 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200435 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200436 event.Set();
437 });
438 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200439 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200440 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200441 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200443 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200444 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 event.Set();
446 });
447 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200448 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 }
450
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200451 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100452 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200453 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
454 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200455 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700456};
457
Noah Richards51db4212019-06-12 06:59:12 -0700458// Simulates simulcast behavior and makes highest stream resolutions divisible
459// by 4.
460class CroppingVideoStreamFactory
461 : public VideoEncoderConfig::VideoStreamFactoryInterface {
462 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200463 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700464
465 private:
466 std::vector<VideoStream> CreateEncoderStreams(
467 int width,
468 int height,
469 const VideoEncoderConfig& encoder_config) override {
470 std::vector<VideoStream> streams = test::CreateVideoStreams(
471 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700472 return streams;
473 }
Noah Richards51db4212019-06-12 06:59:12 -0700474};
475
sprangb1ca0732017-02-01 08:38:12 -0800476class AdaptingFrameForwarder : public test::FrameForwarder {
477 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200478 explicit AdaptingFrameForwarder(TimeController* time_controller)
479 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700480 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800481
482 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200483 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800484 adaptation_enabled_ = enabled;
485 }
486
asaperssonfab67072017-04-04 05:51:49 -0700487 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200488 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800489 return adaptation_enabled_;
490 }
491
Henrik Boström1124ed12021-02-25 10:30:39 +0100492 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
493 // the resolution or frame rate was different than it is currently. If
494 // something else is modified, such as encoder resolutions, but the resolution
495 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700496 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200497 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700498 return last_wants_;
499 }
500
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200501 absl::optional<int> last_sent_width() const { return last_width_; }
502 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800503
sprangb1ca0732017-02-01 08:38:12 -0800504 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200505 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
506 time_controller_->AdvanceTime(TimeDelta::Millis(0));
507
sprangb1ca0732017-02-01 08:38:12 -0800508 int cropped_width = 0;
509 int cropped_height = 0;
510 int out_width = 0;
511 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700512 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000513 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
514 << "w=" << video_frame.width()
515 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700516 if (adapter_.AdaptFrameResolution(
517 video_frame.width(), video_frame.height(),
518 video_frame.timestamp_us() * 1000, &cropped_width,
519 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100520 VideoFrame adapted_frame =
521 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200522 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100523 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200524 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100525 .set_timestamp_ms(99)
526 .set_rotation(kVideoRotation_0)
527 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100528 if (video_frame.has_update_rect()) {
529 adapted_frame.set_update_rect(
530 video_frame.update_rect().ScaleWithFrame(
531 video_frame.width(), video_frame.height(), 0, 0,
532 video_frame.width(), video_frame.height(), out_width,
533 out_height));
534 }
sprangc5d62e22017-04-02 23:53:04 -0700535 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800536 last_width_.emplace(adapted_frame.width());
537 last_height_.emplace(adapted_frame.height());
538 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200539 last_width_ = absl::nullopt;
540 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700541 }
sprangb1ca0732017-02-01 08:38:12 -0800542 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000543 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800544 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800545 last_width_.emplace(video_frame.width());
546 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800547 }
548 }
549
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200550 void OnOutputFormatRequest(int width, int height) {
551 absl::optional<std::pair<int, int>> target_aspect_ratio =
552 std::make_pair(width, height);
553 absl::optional<int> max_pixel_count = width * height;
554 absl::optional<int> max_fps;
555 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
556 max_fps);
557 }
558
sprangb1ca0732017-02-01 08:38:12 -0800559 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
560 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200561 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100562 rtc::VideoSinkWants prev_wants = sink_wants_locked();
563 bool did_adapt =
564 prev_wants.max_pixel_count != wants.max_pixel_count ||
565 prev_wants.target_pixel_count != wants.target_pixel_count ||
566 prev_wants.max_framerate_fps != wants.max_framerate_fps;
567 if (did_adapt) {
568 last_wants_ = prev_wants;
569 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100570 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200571 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800572 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200573
574 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800575 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200576 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
577 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200578 absl::optional<int> last_width_;
579 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800580};
sprangc5d62e22017-04-02 23:53:04 -0700581
Niels Möller213618e2018-07-24 09:29:58 +0200582// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700583class MockableSendStatisticsProxy : public SendStatisticsProxy {
584 public:
585 MockableSendStatisticsProxy(Clock* clock,
586 const VideoSendStream::Config& config,
587 VideoEncoderConfig::ContentType content_type)
588 : SendStatisticsProxy(clock, config, content_type) {}
589
590 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200591 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700592 if (mock_stats_)
593 return *mock_stats_;
594 return SendStatisticsProxy::GetStats();
595 }
596
Niels Möller213618e2018-07-24 09:29:58 +0200597 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200599 if (mock_stats_)
600 return mock_stats_->input_frame_rate;
601 return SendStatisticsProxy::GetInputFrameRate();
602 }
sprangc5d62e22017-04-02 23:53:04 -0700603 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200604 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700605 mock_stats_.emplace(stats);
606 }
607
608 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200609 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700610 mock_stats_.reset();
611 }
612
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
614 on_frame_dropped_ = std::move(callback);
615 }
616
sprangc5d62e22017-04-02 23:53:04 -0700617 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200618 void OnFrameDropped(DropReason reason) override {
619 SendStatisticsProxy::OnFrameDropped(reason);
620 if (on_frame_dropped_)
621 on_frame_dropped_(reason);
622 }
623
Markus Handella3765182020-07-08 13:13:32 +0200624 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200625 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200626 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700627};
628
philipel9b058032020-02-10 11:30:00 +0100629class MockEncoderSelector
630 : public VideoEncoderFactory::EncoderSelectorInterface {
631 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200632 MOCK_METHOD(void,
633 OnCurrentEncoder,
634 (const SdpVideoFormat& format),
635 (override));
636 MOCK_METHOD(absl::optional<SdpVideoFormat>,
637 OnAvailableBitrate,
638 (const DataRate& rate),
639 (override));
640 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100641};
642
perkj803d97f2016-11-01 11:45:46 -0700643} // namespace
644
mflodmancc3d4422017-08-03 08:27:51 -0700645class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700646 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200647 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700648
mflodmancc3d4422017-08-03 08:27:51 -0700649 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700650 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700651 codec_width_(320),
652 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200653 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200654 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200655 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700656 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200657 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700658 video_send_config_,
659 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200660 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700661
662 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700663 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700664 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200665 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800666 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200667 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200668 video_send_config_.rtp.payload_name = "FAKE";
669 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700670
Per512ecb32016-09-23 15:52:06 +0200671 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200672 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200673 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
674 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
675 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100676 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700677
Niels Möllerf1338562018-04-26 09:51:47 +0200678 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800679 }
680
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100681 void ConfigureEncoder(
682 VideoEncoderConfig video_encoder_config,
683 VideoStreamEncoder::BitrateAllocationCallbackType
684 allocation_callback_type =
685 VideoStreamEncoder::BitrateAllocationCallbackType::
686 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700687 if (video_stream_encoder_)
688 video_stream_encoder_->Stop();
689 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200690 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100691 video_send_config_.encoder_settings, allocation_callback_type));
Asa Persson606d3cb2021-10-04 10:07:11 +0200692 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700693 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700694 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200695 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700696 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200697 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700698 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800699 }
700
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100701 void ResetEncoder(const std::string& payload_name,
702 size_t num_streams,
703 size_t num_temporal_layers,
704 unsigned char num_spatial_layers,
705 bool screenshare,
706 VideoStreamEncoder::BitrateAllocationCallbackType
707 allocation_callback_type =
708 VideoStreamEncoder::BitrateAllocationCallbackType::
709 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200710 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800711
712 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200713 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
714 num_streams, &video_encoder_config);
715 for (auto& layer : video_encoder_config.simulcast_layers) {
716 layer.num_temporal_layers = num_temporal_layers;
717 layer.max_framerate = kDefaultFramerate;
718 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100719 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200720 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700721 video_encoder_config.content_type =
722 screenshare ? VideoEncoderConfig::ContentType::kScreen
723 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700724 if (payload_name == "VP9") {
725 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
726 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200727 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700728 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200729 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
730 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700731 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100732 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700733 }
734
sprang57c2fff2017-01-16 06:24:02 -0800735 VideoFrame CreateFrame(int64_t ntp_time_ms,
736 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200737 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200738 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200739 destruction_event, codec_width_, codec_height_))
740 .set_ntp_time_ms(ntp_time_ms)
741 .set_timestamp_ms(99)
742 .set_rotation(kVideoRotation_0)
743 .build();
perkj26091b12016-09-01 01:17:40 -0700744 }
745
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100746 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
747 rtc::Event* destruction_event,
748 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200749 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200750 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200751 destruction_event, codec_width_, codec_height_))
752 .set_ntp_time_ms(ntp_time_ms)
753 .set_timestamp_ms(99)
754 .set_rotation(kVideoRotation_0)
755 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
756 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100757 }
758
sprang57c2fff2017-01-16 06:24:02 -0800759 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200760 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
761 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200762 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200763 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200764 .set_ntp_time_ms(ntp_time_ms)
765 .set_timestamp_ms(ntp_time_ms)
766 .set_rotation(kVideoRotation_0)
767 .build();
perkj803d97f2016-11-01 11:45:46 -0700768 }
769
Evan Shrubsole895556e2020-10-05 09:15:13 +0200770 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200771 return VideoFrame::Builder()
772 .set_video_frame_buffer(NV12Buffer::Create(width, height))
773 .set_ntp_time_ms(ntp_time_ms)
774 .set_timestamp_ms(ntp_time_ms)
775 .set_rotation(kVideoRotation_0)
776 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200777 }
778
Noah Richards51db4212019-06-12 06:59:12 -0700779 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
780 rtc::Event* destruction_event,
781 int width,
782 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200783 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200784 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200785 destruction_event, width, height))
786 .set_ntp_time_ms(ntp_time_ms)
787 .set_timestamp_ms(99)
788 .set_rotation(kVideoRotation_0)
789 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700790 }
791
Evan Shrubsole895556e2020-10-05 09:15:13 +0200792 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
793 rtc::Event* destruction_event,
794 int width,
795 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200796 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200797 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200798 destruction_event, width, height))
799 .set_ntp_time_ms(ntp_time_ms)
800 .set_timestamp_ms(99)
801 .set_rotation(kVideoRotation_0)
802 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200803 }
804
Noah Richards51db4212019-06-12 06:59:12 -0700805 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
806 rtc::Event* destruction_event) const {
807 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
808 codec_height_);
809 }
810
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100811 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200813 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100814
815 video_source_.IncomingCapturedFrame(
816 CreateFrame(1, codec_width_, codec_height_));
817 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200818 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100819 }
820
sprang4847ae62017-06-27 07:06:52 -0700821 void WaitForEncodedFrame(int64_t expected_ntp_time) {
822 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200823 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700824 }
825
826 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
827 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200828 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700829 return ok;
830 }
831
832 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
833 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700835 }
836
837 void ExpectDroppedFrame() {
838 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200839 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700840 }
841
842 bool WaitForFrame(int64_t timeout_ms) {
843 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200844 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700845 return ok;
846 }
847
perkj26091b12016-09-01 01:17:40 -0700848 class TestEncoder : public test::FakeEncoder {
849 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200850 explicit TestEncoder(TimeController* time_controller)
851 : FakeEncoder(time_controller->GetClock()),
852 time_controller_(time_controller) {
853 RTC_DCHECK(time_controller_);
854 }
perkj26091b12016-09-01 01:17:40 -0700855
perkjfa10b552016-10-02 23:45:26 -0700856 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200857 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700858 block_next_encode_ = true;
859 }
860
Erik Språngaed30702018-11-05 12:57:17 +0100861 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200862 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200863 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100864 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100865 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100866 info.scaling_settings = VideoEncoder::ScalingSettings(
867 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100868 }
869 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100870 for (int i = 0; i < kMaxSpatialLayers; ++i) {
871 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100872 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100873 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100874 for (int tid = 0; tid < num_layers; ++tid)
875 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100876 }
877 }
Erik Språngaed30702018-11-05 12:57:17 +0100878 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200879
880 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100881 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200882 info.apply_alignment_to_all_simulcast_layers =
883 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200884 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800885 if (is_qp_trusted_.has_value()) {
886 info.is_qp_trusted = is_qp_trusted_;
887 }
Erik Språngaed30702018-11-05 12:57:17 +0100888 return info;
kthelgason876222f2016-11-29 01:44:11 -0800889 }
890
Erik Språngb7cb7b52019-02-26 15:52:33 +0100891 int32_t RegisterEncodeCompleteCallback(
892 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200893 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100894 encoded_image_callback_ = callback;
895 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
896 }
897
perkjfa10b552016-10-02 23:45:26 -0700898 void ContinueEncode() { continue_encode_event_.Set(); }
899
900 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
901 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200902 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700903 EXPECT_EQ(timestamp_, timestamp);
904 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
905 }
906
kthelgason2fc52542017-03-03 00:24:41 -0800907 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200908 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800909 quality_scaling_ = b;
910 }
kthelgasonad9010c2017-02-14 00:46:51 -0800911
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100912 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200913 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100914 requested_resolution_alignment_ = requested_resolution_alignment;
915 }
916
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200917 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
918 MutexLock lock(&local_mutex_);
919 apply_alignment_to_all_simulcast_layers_ = b;
920 }
921
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100922 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200923 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100924 is_hardware_accelerated_ = is_hardware_accelerated;
925 }
926
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100927 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
928 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200929 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100930 temporal_layers_supported_[spatial_idx] = supported;
931 }
932
Sergey Silkin6456e352019-07-08 17:56:40 +0200933 void SetResolutionBitrateLimits(
934 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200936 resolution_bitrate_limits_ = thresholds;
937 }
938
sprangfe627f32017-03-29 08:24:59 -0700939 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200940 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700941 force_init_encode_failed_ = force_failure;
942 }
943
Niels Möller6bb5ab92019-01-11 11:11:10 +0100944 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200945 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100946 rate_factor_ = rate_factor;
947 }
948
Erik Språngd7329ca2019-02-21 21:19:53 +0100949 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200950 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100951 return last_framerate_;
952 }
953
Erik Språngd7329ca2019-02-21 21:19:53 +0100954 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200955 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100956 return last_update_rect_;
957 }
958
Niels Möller87e2d782019-03-07 10:18:23 +0100959 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200960 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100961 return last_frame_types_;
962 }
963
964 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100965 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100966 keyframe ? VideoFrameType::kVideoFrameKey
967 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100968 {
Markus Handella3765182020-07-08 13:13:32 +0200969 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100970 last_frame_types_ = frame_type;
971 }
Niels Möllerb859b322019-03-07 12:40:01 +0100972 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100973 }
974
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100975 void InjectEncodedImage(const EncodedImage& image,
976 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200977 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100978 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100979 }
980
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200981 void SetEncodedImageData(
982 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200983 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200984 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200985 }
986
Erik Språngd7329ca2019-02-21 21:19:53 +0100987 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200988 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100989 expect_null_frame_ = true;
990 }
991
Erik Språng5056af02019-09-02 15:53:11 +0200992 absl::optional<VideoEncoder::RateControlParameters>
993 GetAndResetLastRateControlSettings() {
994 auto settings = last_rate_control_settings_;
995 last_rate_control_settings_.reset();
996 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100997 }
998
Henrik Boström56db9ff2021-03-24 09:06:45 +0100999 int GetLastInputWidth() const {
1000 MutexLock lock(&local_mutex_);
1001 return last_input_width_;
1002 }
1003
1004 int GetLastInputHeight() const {
1005 MutexLock lock(&local_mutex_);
1006 return last_input_height_;
1007 }
1008
Evan Shrubsole895556e2020-10-05 09:15:13 +02001009 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1010 MutexLock lock(&local_mutex_);
1011 return last_input_pixel_format_;
1012 }
1013
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001014 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001015 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001016 return num_set_rates_;
1017 }
1018
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001019 void SetPreferredPixelFormats(
1020 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1021 pixel_formats) {
1022 MutexLock lock(&local_mutex_);
1023 preferred_pixel_formats_ = std::move(pixel_formats);
1024 }
1025
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001026 void SetIsQpTrusted(absl::optional<bool> trusted) {
1027 MutexLock lock(&local_mutex_);
1028 is_qp_trusted_ = trusted;
1029 }
1030
perkjfa10b552016-10-02 23:45:26 -07001031 private:
perkj26091b12016-09-01 01:17:40 -07001032 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001033 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001034 bool block_encode;
1035 {
Markus Handella3765182020-07-08 13:13:32 +02001036 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001037 if (expect_null_frame_) {
1038 EXPECT_EQ(input_image.timestamp(), 0u);
1039 EXPECT_EQ(input_image.width(), 1);
1040 last_frame_types_ = *frame_types;
1041 expect_null_frame_ = false;
1042 } else {
1043 EXPECT_GT(input_image.timestamp(), timestamp_);
1044 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1045 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1046 }
perkj26091b12016-09-01 01:17:40 -07001047
1048 timestamp_ = input_image.timestamp();
1049 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001050 last_input_width_ = input_image.width();
1051 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001052 block_encode = block_next_encode_;
1053 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001054 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001055 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001056 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001057 }
Niels Möllerb859b322019-03-07 12:40:01 +01001058 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001059 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001060 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001061
perkj26091b12016-09-01 01:17:40 -07001062 return result;
1063 }
1064
Niels Möller08ae7ce2020-09-23 15:58:12 +02001065 CodecSpecificInfo EncodeHook(
1066 EncodedImage& encoded_image,
1067 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001068 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001069 {
1070 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001071 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001072 }
1073 MutexLock lock(&local_mutex_);
1074 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001075 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001076 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001077 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001078 }
1079
sprangfe627f32017-03-29 08:24:59 -07001080 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001081 const Settings& settings) override {
1082 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001083
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001085 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001086
Erik Språng82fad3d2018-03-21 09:57:23 +01001087 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001088 // Simulate setting up temporal layers, in order to validate the life
1089 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001090 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001091 frame_buffer_controller_ =
1092 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001093 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001094 if (force_init_encode_failed_) {
1095 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001096 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001097 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001098
Erik Språngb7cb7b52019-02-26 15:52:33 +01001099 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001100 return res;
1101 }
1102
Erik Språngb7cb7b52019-02-26 15:52:33 +01001103 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001104 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001105 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1106 initialized_ = EncoderState::kUninitialized;
1107 return FakeEncoder::Release();
1108 }
1109
Erik Språng16cb8f52019-04-12 13:59:09 +02001110 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001112 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001113 VideoBitrateAllocation adjusted_rate_allocation;
1114 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1115 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001116 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001117 adjusted_rate_allocation.SetBitrate(
1118 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001119 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001120 rate_factor_));
1121 }
1122 }
1123 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001124 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001125 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001126 RateControlParameters adjusted_paramters = parameters;
1127 adjusted_paramters.bitrate = adjusted_rate_allocation;
1128 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001129 }
1130
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001131 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001132 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001133 enum class EncoderState {
1134 kUninitialized,
1135 kInitializationFailed,
1136 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001137 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1138 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001139 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001140 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1141 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1142 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1143 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1144 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1145 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001146 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1147 false;
Markus Handella3765182020-07-08 13:13:32 +02001148 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001149 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1150 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001151 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001152 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001153 absl::optional<bool>
1154 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001155 local_mutex_);
1156 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1157 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1158 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001159 absl::optional<VideoEncoder::RateControlParameters>
1160 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001161 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1162 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001163 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001164 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001165 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1166 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001167 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001168 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001169 RTC_GUARDED_BY(local_mutex_);
1170 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001171 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1172 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001173 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1174 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001175 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001176 };
1177
mflodmancc3d4422017-08-03 08:27:51 -07001178 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001179 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001180 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1181 : time_controller_(time_controller), test_encoder_(test_encoder) {
1182 RTC_DCHECK(time_controller_);
1183 }
perkj26091b12016-09-01 01:17:40 -07001184
perkj26091b12016-09-01 01:17:40 -07001185 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001186 EXPECT_TRUE(
1187 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1188 }
1189
1190 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1191 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001192 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001193 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001194 return false;
perkj26091b12016-09-01 01:17:40 -07001195 {
Markus Handella3765182020-07-08 13:13:32 +02001196 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001197 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001198 }
1199 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001200 return true;
perkj26091b12016-09-01 01:17:40 -07001201 }
1202
sprangb1ca0732017-02-01 08:38:12 -08001203 void WaitForEncodedFrame(uint32_t expected_width,
1204 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001205 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001206 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001207 }
1208
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001209 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001210 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001211 uint32_t width = 0;
1212 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001213 {
Markus Handella3765182020-07-08 13:13:32 +02001214 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001215 width = last_width_;
1216 height = last_height_;
1217 }
1218 EXPECT_EQ(expected_height, height);
1219 EXPECT_EQ(expected_width, width);
1220 }
1221
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001222 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1223 VideoRotation rotation;
1224 {
Markus Handella3765182020-07-08 13:13:32 +02001225 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001226 rotation = last_rotation_;
1227 }
1228 EXPECT_EQ(expected_rotation, rotation);
1229 }
1230
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001231 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001232
sprangc5d62e22017-04-02 23:53:04 -07001233 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001234 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1235 bool ret = encoded_frame_event_.Wait(timeout_ms);
1236 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1237 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001238 }
1239
perkj26091b12016-09-01 01:17:40 -07001240 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001241 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001242 expect_frames_ = false;
1243 }
1244
asaperssonfab67072017-04-04 05:51:49 -07001245 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001246 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001247 return number_of_reconfigurations_;
1248 }
1249
asaperssonfab67072017-04-04 05:51:49 -07001250 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001251 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001252 return min_transmit_bitrate_bps_;
1253 }
1254
Erik Språngd7329ca2019-02-21 21:19:53 +01001255 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001256 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001257 num_expected_layers_ = num_layers;
1258 }
1259
Erik Språngb7cb7b52019-02-26 15:52:33 +01001260 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001261 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001262 return last_capture_time_ms_;
1263 }
1264
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001265 const EncodedImage& GetLastEncodedImage() {
1266 MutexLock lock(&mutex_);
1267 return last_encoded_image_;
1268 }
1269
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001270 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001271 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001272 return std::move(last_encoded_image_data_);
1273 }
1274
Per Kjellanderdcef6412020-10-07 15:09:05 +02001275 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1276 MutexLock lock(&mutex_);
1277 return last_bitrate_allocation_;
1278 }
1279
1280 int number_of_bitrate_allocations() const {
1281 MutexLock lock(&mutex_);
1282 return number_of_bitrate_allocations_;
1283 }
1284
Per Kjellandera9434842020-10-15 17:53:22 +02001285 VideoLayersAllocation GetLastVideoLayersAllocation() {
1286 MutexLock lock(&mutex_);
1287 return last_layers_allocation_;
1288 }
1289
1290 int number_of_layers_allocations() const {
1291 MutexLock lock(&mutex_);
1292 return number_of_layers_allocations_;
1293 }
1294
perkj26091b12016-09-01 01:17:40 -07001295 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001296 Result OnEncodedImage(
1297 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001298 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001299 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001300 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001301 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001302 last_encoded_image_data_ = std::vector<uint8_t>(
1303 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001304 uint32_t timestamp = encoded_image.Timestamp();
1305 if (last_timestamp_ != timestamp) {
1306 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001307 last_width_ = encoded_image._encodedWidth;
1308 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001309 } else {
1310 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001311 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1312 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001313 }
1314 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001315 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001316 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001317 if (num_received_layers_ == num_expected_layers_) {
1318 encoded_frame_event_.Set();
1319 }
sprangb1ca0732017-02-01 08:38:12 -08001320 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001321 }
1322
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001323 void OnEncoderConfigurationChanged(
1324 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001325 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001326 VideoEncoderConfig::ContentType content_type,
1327 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001328 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001329 ++number_of_reconfigurations_;
1330 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1331 }
1332
Per Kjellanderdcef6412020-10-07 15:09:05 +02001333 void OnBitrateAllocationUpdated(
1334 const VideoBitrateAllocation& allocation) override {
1335 MutexLock lock(&mutex_);
1336 ++number_of_bitrate_allocations_;
1337 last_bitrate_allocation_ = allocation;
1338 }
1339
Per Kjellandera9434842020-10-15 17:53:22 +02001340 void OnVideoLayersAllocationUpdated(
1341 VideoLayersAllocation allocation) override {
1342 MutexLock lock(&mutex_);
1343 ++number_of_layers_allocations_;
1344 last_layers_allocation_ = allocation;
1345 rtc::StringBuilder log;
1346 for (const auto& layer : allocation.active_spatial_layers) {
1347 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1348 << "[";
1349 for (const auto target_bitrate :
1350 layer.target_bitrate_per_temporal_layer) {
1351 log << target_bitrate.kbps() << ",";
1352 }
1353 log << "]";
1354 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001355 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001356 }
1357
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001358 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001359 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001360 TestEncoder* test_encoder_;
1361 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001362 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001363 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001364 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001365 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001366 uint32_t last_height_ = 0;
1367 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001368 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001369 size_t num_expected_layers_ = 1;
1370 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001371 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001372 int number_of_reconfigurations_ = 0;
1373 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001374 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1375 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001376 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1377 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001378 };
1379
Sergey Silkin5ee69672019-07-02 14:18:34 +02001380 class VideoBitrateAllocatorProxyFactory
1381 : public VideoBitrateAllocatorFactory {
1382 public:
1383 VideoBitrateAllocatorProxyFactory()
1384 : bitrate_allocator_factory_(
1385 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1386
1387 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1388 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001389 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001390 codec_config_ = codec;
1391 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1392 }
1393
1394 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001395 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001396 return codec_config_;
1397 }
1398
1399 private:
1400 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1401
Markus Handella3765182020-07-08 13:13:32 +02001402 mutable Mutex mutex_;
1403 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001404 };
1405
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001406 Clock* clock() { return time_controller_.GetClock(); }
1407 void AdvanceTime(TimeDelta duration) {
1408 time_controller_.AdvanceTime(duration);
1409 }
1410
1411 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1412
1413 protected:
1414 virtual TaskQueueFactory* GetTaskQueueFactory() {
1415 return time_controller_.GetTaskQueueFactory();
1416 }
1417
1418 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001419 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001420 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001421 int codec_width_;
1422 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001423 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001424 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001425 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001426 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001427 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001428 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001429 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001430 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001431};
1432
mflodmancc3d4422017-08-03 08:27:51 -07001433TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001435 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001436 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001437 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001438 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001439 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001440 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001441}
1442
mflodmancc3d4422017-08-03 08:27:51 -07001443TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001444 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001445 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001446 // The encoder will cache up to one frame for a short duration. Adding two
1447 // frames means that the first frame will be dropped and the second frame will
1448 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001449 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001450 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001451 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001452 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001453
Henrik Boström381d1092020-05-12 18:49:07 +02001454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001455 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001456
Sebastian Janssona3177052018-04-10 13:05:49 +02001457 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001458 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001459 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1460
1461 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001462 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001463}
1464
mflodmancc3d4422017-08-03 08:27:51 -07001465TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001467 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001468 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001469 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001470
Henrik Boström381d1092020-05-12 18:49:07 +02001471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001472 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1473
Sebastian Janssona3177052018-04-10 13:05:49 +02001474 // The encoder will cache up to one frame for a short duration. Adding two
1475 // frames means that the first frame will be dropped and the second frame will
1476 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001477 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001478 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001479
Henrik Boström381d1092020-05-12 18:49:07 +02001480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001481 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001482 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001483 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1484 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001485 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001486}
1487
mflodmancc3d4422017-08-03 08:27:51 -07001488TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001490 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001491 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001492 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001493
1494 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001496
perkja49cbd32016-09-16 07:53:41 -07001497 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001498 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001499 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001500}
1501
mflodmancc3d4422017-08-03 08:27:51 -07001502TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001504 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001505
perkja49cbd32016-09-16 07:53:41 -07001506 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001507 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001508
mflodmancc3d4422017-08-03 08:27:51 -07001509 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001510 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001511 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001512 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1513 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001514}
1515
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001516class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1517 public:
1518 VideoStreamEncoderBlockedTest() {}
1519
1520 TaskQueueFactory* GetTaskQueueFactory() override {
1521 return task_queue_factory_.get();
1522 }
1523
1524 private:
1525 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1526 CreateDefaultTaskQueueFactory();
1527};
1528
1529TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001531 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001532
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001533 int dropped_count = 0;
1534 stats_proxy_->SetDroppedFrameCallback(
1535 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1536 ++dropped_count;
1537 });
1538
perkj26091b12016-09-01 01:17:40 -07001539 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001540 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001541 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001542 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1543 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001544 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1545 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001546 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001547 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001548
mflodmancc3d4422017-08-03 08:27:51 -07001549 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001550
1551 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001552}
1553
Henrik Boström56db9ff2021-03-24 09:06:45 +01001554TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001555 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001556 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001557
1558 rtc::Event frame_destroyed_event;
1559 video_source_.IncomingCapturedFrame(
1560 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001561 WaitForEncodedFrame(1);
1562 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1563 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001564 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1565 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001566 video_stream_encoder_->Stop();
1567}
1568
Henrik Boström56db9ff2021-03-24 09:06:45 +01001569TEST_F(VideoStreamEncoderTest,
1570 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001571 // Use the cropping factory.
1572 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001573 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001574 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1575 kMaxPayloadLength);
1576 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1577
1578 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001580 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001581 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1582 WaitForEncodedFrame(1);
1583 // The encoder will have been configured once.
1584 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001585 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1586 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001587
1588 // Now send in a fake frame that needs to be cropped as the width/height
1589 // aren't divisible by 4 (see CreateEncoderStreams above).
1590 rtc::Event frame_destroyed_event;
1591 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1592 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001593 WaitForEncodedFrame(2);
1594 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1595 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001596 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1597 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001598 video_stream_encoder_->Stop();
1599}
1600
Evan Shrubsole895556e2020-10-05 09:15:13 +02001601TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001603 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001604
1605 video_source_.IncomingCapturedFrame(
1606 CreateNV12Frame(1, codec_width_, codec_height_));
1607 WaitForEncodedFrame(1);
1608 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1609 fake_encoder_.GetLastInputPixelFormat());
1610 video_stream_encoder_->Stop();
1611}
1612
Henrik Boström56db9ff2021-03-24 09:06:45 +01001613TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001614 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001615 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001616
1617 fake_encoder_.SetPreferredPixelFormats({});
1618
1619 rtc::Event frame_destroyed_event;
1620 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1621 1, &frame_destroyed_event, codec_width_, codec_height_));
1622 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001623 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001624 fake_encoder_.GetLastInputPixelFormat());
1625 video_stream_encoder_->Stop();
1626}
1627
Henrik Boström56db9ff2021-03-24 09:06:45 +01001628TEST_F(VideoStreamEncoderTest,
1629 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001631 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001632
1633 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1634
1635 rtc::Event frame_destroyed_event;
1636 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1637 1, &frame_destroyed_event, codec_width_, codec_height_));
1638 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001639 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001640 fake_encoder_.GetLastInputPixelFormat());
1641 video_stream_encoder_->Stop();
1642}
1643
Henrik Boström56db9ff2021-03-24 09:06:45 +01001644TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001646 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001647
1648 // Fake NV12 native frame does not allow mapping to I444.
1649 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1650
1651 rtc::Event frame_destroyed_event;
1652 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1653 1, &frame_destroyed_event, codec_width_, codec_height_));
1654 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001655 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001656 fake_encoder_.GetLastInputPixelFormat());
1657 video_stream_encoder_->Stop();
1658}
1659
Henrik Boström56db9ff2021-03-24 09:06:45 +01001660TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001662 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001663
1664 rtc::Event frame_destroyed_event;
1665 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1666 1, &frame_destroyed_event, codec_width_, codec_height_));
1667 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001668 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001669 fake_encoder_.GetLastInputPixelFormat());
1670 video_stream_encoder_->Stop();
1671}
1672
Ying Wang9b881ab2020-02-07 14:29:32 +01001673TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001676 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1677 WaitForEncodedFrame(1);
1678
Henrik Boström381d1092020-05-12 18:49:07 +02001679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001680 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001681 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1682 // frames. Adding two frames means that the first frame will be dropped and
1683 // the second frame will be sent to the encoder.
1684 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1685 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1686 WaitForEncodedFrame(3);
1687 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1688 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1689 WaitForEncodedFrame(5);
1690 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1691 video_stream_encoder_->Stop();
1692}
1693
mflodmancc3d4422017-08-03 08:27:51 -07001694TEST_F(VideoStreamEncoderTest,
1695 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001698 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001699
1700 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001701 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001702 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001703 // The encoder will have been configured once when the first frame is
1704 // received.
1705 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001706
1707 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001708 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001709 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001710 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001711 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001712
1713 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001714 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001715 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001716 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001717 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001718
mflodmancc3d4422017-08-03 08:27:51 -07001719 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001720}
1721
mflodmancc3d4422017-08-03 08:27:51 -07001722TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001724 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001725
1726 // Capture a frame and wait for it to synchronize with the encoder thread.
1727 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001728 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001729 // The encoder will have been configured once.
1730 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001731 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1732 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001733
1734 codec_width_ *= 2;
1735 codec_height_ *= 2;
1736 // Capture a frame with a higher resolution and wait for it to synchronize
1737 // with the encoder thread.
1738 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001739 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001740 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1741 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001742 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001743
mflodmancc3d4422017-08-03 08:27:51 -07001744 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001745}
1746
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001747TEST_F(VideoStreamEncoderTest,
1748 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001750 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001751
1752 // Capture a frame and wait for it to synchronize with the encoder thread.
1753 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1754 WaitForEncodedFrame(1);
1755
1756 VideoEncoderConfig video_encoder_config;
1757 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1758 // Changing the max payload data length recreates encoder.
1759 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1760 kMaxPayloadLength / 2);
1761
1762 // Capture a frame and wait for it to synchronize with the encoder thread.
1763 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1764 WaitForEncodedFrame(2);
1765 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1766
1767 video_stream_encoder_->Stop();
1768}
1769
Sergey Silkin5ee69672019-07-02 14:18:34 +02001770TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001773
1774 VideoEncoderConfig video_encoder_config;
1775 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001776 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1777 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001778 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1779 kMaxPayloadLength);
1780
1781 // Capture a frame and wait for it to synchronize with the encoder thread.
1782 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1783 WaitForEncodedFrame(1);
1784 // The encoder will have been configured once when the first frame is
1785 // received.
1786 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001787 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001788 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001789 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001790 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1791
Sergey Silkin6456e352019-07-08 17:56:40 +02001792 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1793 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001794 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1795 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001796 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1797 kMaxPayloadLength);
1798
1799 // Capture a frame and wait for it to synchronize with the encoder thread.
1800 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1801 WaitForEncodedFrame(2);
1802 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1803 // Bitrate limits have changed - rate allocator should be reconfigured,
1804 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001805 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001806 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001807 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001808 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001809 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001810
1811 video_stream_encoder_->Stop();
1812}
1813
Sergey Silkin6456e352019-07-08 17:56:40 +02001814TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001815 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001817 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001818
Sergey Silkincd02eba2020-01-20 14:48:40 +01001819 const uint32_t kMinEncBitrateKbps = 100;
1820 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001821 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001822 /*frame_size_pixels=*/codec_width_ * codec_height_,
1823 /*min_start_bitrate_bps=*/0,
1824 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1825 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001826 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1827
Sergey Silkincd02eba2020-01-20 14:48:40 +01001828 VideoEncoderConfig video_encoder_config;
1829 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1830 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1831 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1832 (kMinEncBitrateKbps + 1) * 1000;
1833 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1834 kMaxPayloadLength);
1835
1836 // When both encoder and app provide bitrate limits, the intersection of
1837 // provided sets should be used.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 EXPECT_EQ(kMaxEncBitrateKbps,
1841 bitrate_allocator_factory_.codec_config().maxBitrate);
1842 EXPECT_EQ(kMinEncBitrateKbps + 1,
1843 bitrate_allocator_factory_.codec_config().minBitrate);
1844
1845 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1846 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1847 (kMinEncBitrateKbps - 1) * 1000;
1848 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1849 kMaxPayloadLength);
1850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001851 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001852 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001853 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001854 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001855 bitrate_allocator_factory_.codec_config().minBitrate);
1856
Sergey Silkincd02eba2020-01-20 14:48:40 +01001857 video_stream_encoder_->Stop();
1858}
1859
1860TEST_F(VideoStreamEncoderTest,
1861 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001863 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001864
1865 const uint32_t kMinAppBitrateKbps = 100;
1866 const uint32_t kMaxAppBitrateKbps = 200;
1867 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1868 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1869 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1870 /*frame_size_pixels=*/codec_width_ * codec_height_,
1871 /*min_start_bitrate_bps=*/0,
1872 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1873 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1874 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1875
1876 VideoEncoderConfig video_encoder_config;
1877 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1878 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1879 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1880 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001881 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1882 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001883
Sergey Silkincd02eba2020-01-20 14:48:40 +01001884 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1885 WaitForEncodedFrame(1);
1886 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001887 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001888 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001889 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001890
1891 video_stream_encoder_->Stop();
1892}
1893
1894TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001895 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001896 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001897 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001898
1899 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001901 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001902 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001903 fake_encoder_.SetResolutionBitrateLimits(
1904 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1905
1906 VideoEncoderConfig video_encoder_config;
1907 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1908 video_encoder_config.max_bitrate_bps = 0;
1909 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1910 kMaxPayloadLength);
1911
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001912 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001913 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1914 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001915 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1916 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001917 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1918 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1919
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001920 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001921 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1922 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001923 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1924 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001925 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1926 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1927
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001928 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001929 // encoder for 360p should be used.
1930 video_source_.IncomingCapturedFrame(
1931 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1932 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001933 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1934 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001935 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1936 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1937
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001938 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001939 // ignored.
1940 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1941 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1943 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001944 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1945 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001946 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1947 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001948 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1949 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1950
1951 // Resolution lower than 270p. The max bitrate limit recommended by encoder
1952 // for 270p should be used.
1953 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
1954 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001955 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1956 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001957 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1958 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1959
1960 video_stream_encoder_->Stop();
1961}
1962
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001963TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001965 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001966
1967 VideoEncoderConfig video_encoder_config;
1968 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1969 video_encoder_config.max_bitrate_bps = 0;
1970 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1971 kMaxPayloadLength);
1972
1973 // Encode 720p frame to get the default encoder target bitrate.
1974 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
1975 WaitForEncodedFrame(1);
1976 const uint32_t kDefaultTargetBitrateFor720pKbps =
1977 bitrate_allocator_factory_.codec_config()
1978 .simulcastStream[0]
1979 .targetBitrate;
1980
1981 // Set the max recommended encoder bitrate to something lower than the default
1982 // target bitrate.
1983 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1984 1280 * 720, 10 * 1000, 10 * 1000,
1985 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
1986 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1987
1988 // Change resolution to trigger encoder reinitialization.
1989 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1990 WaitForEncodedFrame(2);
1991 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
1992 WaitForEncodedFrame(3);
1993
1994 // Ensure the target bitrate is capped by the max bitrate.
1995 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
1996 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
1997 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
1998 .simulcastStream[0]
1999 .targetBitrate *
2000 1000,
2001 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2002
2003 video_stream_encoder_->Stop();
2004}
2005
Åsa Perssona7e34d32021-01-20 15:36:13 +01002006TEST_F(VideoStreamEncoderTest,
2007 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2008 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2009 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2010 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2011 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2012 fake_encoder_.SetResolutionBitrateLimits(
2013 {kEncoderLimits270p, kEncoderLimits360p});
2014
2015 // Two streams, highest stream active.
2016 VideoEncoderConfig config;
2017 const int kNumStreams = 2;
2018 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2019 config.max_bitrate_bps = 0;
2020 config.simulcast_layers[0].active = false;
2021 config.simulcast_layers[1].active = true;
2022 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002023 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002024 "VP8", /*max qp*/ 56, /*screencast*/ false,
2025 /*screenshare enabled*/ false);
2026 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2027
2028 // The encoder bitrate limits for 270p should be used.
2029 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2030 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002031 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002032 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002033 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002034 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002035 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002036
2037 // The encoder bitrate limits for 360p should be used.
2038 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2039 EXPECT_FALSE(WaitForFrame(1000));
2040 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002041 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002042 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002043 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002044
2045 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2046 video_source_.IncomingCapturedFrame(
2047 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2048 EXPECT_FALSE(WaitForFrame(1000));
2049 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002050 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002051 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002052 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002053
2054 // Resolution higher than 360p. Encoder limits should be ignored.
2055 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2056 EXPECT_FALSE(WaitForFrame(1000));
2057 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002058 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002059 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002060 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002061 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002062 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002063 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002064 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002065
2066 // Resolution lower than 270p. The encoder limits for 270p should be used.
2067 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2068 EXPECT_FALSE(WaitForFrame(1000));
2069 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002070 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002071 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002072 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002073
2074 video_stream_encoder_->Stop();
2075}
2076
2077TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002078 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2079 // Two streams, highest stream active.
2080 VideoEncoderConfig config;
2081 const int kNumStreams = 2;
2082 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2083 config.max_bitrate_bps = 0;
2084 config.simulcast_layers[0].active = false;
2085 config.simulcast_layers[1].active = true;
2086 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002087 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002088 "VP8", /*max qp*/ 56, /*screencast*/ false,
2089 /*screenshare enabled*/ false);
2090 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2091
2092 // Default bitrate limits for 270p should be used.
2093 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2094 kDefaultLimits270p =
2095 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002096 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002097 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2098 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002099 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002100 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002101 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002102 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002103 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002104
2105 // Default bitrate limits for 360p should be used.
2106 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2107 kDefaultLimits360p =
2108 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002109 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002110 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2111 EXPECT_FALSE(WaitForFrame(1000));
2112 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002113 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002114 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002115 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002116
2117 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2118 video_source_.IncomingCapturedFrame(
2119 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2120 EXPECT_FALSE(WaitForFrame(1000));
2121 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002122 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002123 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002124 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002125
2126 // Default bitrate limits for 540p should be used.
2127 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2128 kDefaultLimits540p =
2129 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002130 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002131 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2132 EXPECT_FALSE(WaitForFrame(1000));
2133 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002134 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002135 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002136 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002137
2138 video_stream_encoder_->Stop();
2139}
2140
2141TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002142 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2143 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2144 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2145 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2146 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2147 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2148 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2149 fake_encoder_.SetResolutionBitrateLimits(
2150 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2151
2152 // Three streams, middle stream active.
2153 VideoEncoderConfig config;
2154 const int kNumStreams = 3;
2155 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2156 config.simulcast_layers[0].active = false;
2157 config.simulcast_layers[1].active = true;
2158 config.simulcast_layers[2].active = false;
2159 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002160 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002161 "VP8", /*max qp*/ 56, /*screencast*/ false,
2162 /*screenshare enabled*/ false);
2163 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2164
2165 // The encoder bitrate limits for 360p should be used.
2166 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2167 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002168 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002169 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002170 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002171 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002172 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002173
2174 // The encoder bitrate limits for 270p should be used.
2175 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2176 EXPECT_FALSE(WaitForFrame(1000));
2177 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002178 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002179 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002180 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002181
2182 video_stream_encoder_->Stop();
2183}
2184
2185TEST_F(VideoStreamEncoderTest,
2186 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2187 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2188 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2189 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2190 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2191 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2192 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2193 fake_encoder_.SetResolutionBitrateLimits(
2194 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2195
2196 // Three streams, lowest stream active.
2197 VideoEncoderConfig config;
2198 const int kNumStreams = 3;
2199 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2200 config.simulcast_layers[0].active = true;
2201 config.simulcast_layers[1].active = false;
2202 config.simulcast_layers[2].active = false;
2203 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002204 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002205 "VP8", /*max qp*/ 56, /*screencast*/ false,
2206 /*screenshare enabled*/ false);
2207 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2208
2209 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2210 // on lowest stream, limits for 270p should not be used
2211 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2212 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002215 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002217 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002218
2219 video_stream_encoder_->Stop();
2220}
2221
2222TEST_F(VideoStreamEncoderTest,
2223 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2224 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2225 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2226 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2227 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2228 fake_encoder_.SetResolutionBitrateLimits(
2229 {kEncoderLimits270p, kEncoderLimits360p});
2230 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2231
2232 // Two streams, highest stream active.
2233 VideoEncoderConfig config;
2234 const int kNumStreams = 2;
2235 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2236 config.simulcast_layers[0].active = false;
2237 config.simulcast_layers[1].active = true;
2238 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2239 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002240 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241 "VP8", /*max qp*/ 56, /*screencast*/ false,
2242 /*screenshare enabled*/ false);
2243 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2244
2245 // The encoder bitrate limits for 270p should be used.
2246 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2247 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002248 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002249 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002250 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002251 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002252 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002253
2254 // The max configured bitrate is less than the encoder limit for 360p.
2255 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2256 EXPECT_FALSE(WaitForFrame(1000));
2257 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002258 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002259 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002260 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002261
2262 video_stream_encoder_->Stop();
2263}
2264
mflodmancc3d4422017-08-03 08:27:51 -07002265TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002266 EXPECT_TRUE(video_source_.has_sinks());
2267 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002268 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002269 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002270 EXPECT_FALSE(video_source_.has_sinks());
2271 EXPECT_TRUE(new_video_source.has_sinks());
2272
mflodmancc3d4422017-08-03 08:27:51 -07002273 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002274}
2275
mflodmancc3d4422017-08-03 08:27:51 -07002276TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002277 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002278 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002279 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002280 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002281}
2282
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002283class ResolutionAlignmentTest
2284 : public VideoStreamEncoderTest,
2285 public ::testing::WithParamInterface<
2286 ::testing::tuple<int, std::vector<double>>> {
2287 public:
2288 ResolutionAlignmentTest()
2289 : requested_alignment_(::testing::get<0>(GetParam())),
2290 scale_factors_(::testing::get<1>(GetParam())) {}
2291
2292 protected:
2293 const int requested_alignment_;
2294 const std::vector<double> scale_factors_;
2295};
2296
2297INSTANTIATE_TEST_SUITE_P(
2298 AlignmentAndScaleFactors,
2299 ResolutionAlignmentTest,
2300 ::testing::Combine(
2301 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2302 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2303 std::vector<double>{-1.0, -1.0},
2304 std::vector<double>{-1.0, -1.0, -1.0},
2305 std::vector<double>{4.0, 2.0, 1.0},
2306 std::vector<double>{9999.0, -1.0, 1.0},
2307 std::vector<double>{3.99, 2.01, 1.0},
2308 std::vector<double>{4.9, 1.7, 1.25},
2309 std::vector<double>{10.0, 4.0, 3.0},
2310 std::vector<double>{1.75, 3.5},
2311 std::vector<double>{1.5, 2.5},
2312 std::vector<double>{1.3, 1.0})));
2313
2314TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2315 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002316 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002317 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2318 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2319
2320 // Fill config with the scaling factor by which to reduce encoding size.
2321 const int num_streams = scale_factors_.size();
2322 VideoEncoderConfig config;
2323 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2324 for (int i = 0; i < num_streams; ++i) {
2325 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2326 }
2327 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002328 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002329 "VP8", /*max qp*/ 56, /*screencast*/ false,
2330 /*screenshare enabled*/ false);
2331 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2332
Henrik Boström381d1092020-05-12 18:49:07 +02002333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002334 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2335 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002336 // Wait for all layers before triggering event.
2337 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002338
2339 // On the 1st frame, we should have initialized the encoder and
2340 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002341 int64_t timestamp_ms = kFrameIntervalMs;
2342 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2343 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002345
2346 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2347 // (It's up the to the encoder to potentially drop the previous frame,
2348 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002349 timestamp_ms += kFrameIntervalMs;
2350 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2351 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002352 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002353
Asa Persson606d3cb2021-10-04 10:07:11 +02002354 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002355 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2356 // Frame size should be a multiple of the requested alignment.
2357 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2358 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2359 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2360 // Aspect ratio should match.
2361 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2362 codec.height * codec.simulcastStream[i].width);
2363 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002364
2365 video_stream_encoder_->Stop();
2366}
2367
Jonathan Yubc771b72017-12-08 17:04:29 -08002368TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2369 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002370 const int kWidth = 1280;
2371 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002372
2373 // We rely on the automatic resolution adaptation, but we handle framerate
2374 // adaptation manually by mocking the stats proxy.
2375 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002376
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002377 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002379 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002380 video_stream_encoder_->SetSource(&video_source_,
2381 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002382 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002383 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002384 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002385 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2386
Jonathan Yubc771b72017-12-08 17:04:29 -08002387 // Adapt down as far as possible.
2388 rtc::VideoSinkWants last_wants;
2389 int64_t t = 1;
2390 int loop_count = 0;
2391 do {
2392 ++loop_count;
2393 last_wants = video_source_.sink_wants();
2394
2395 // Simulate the framerate we've been asked to adapt to.
2396 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2397 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2398 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2399 mock_stats.input_frame_rate = fps;
2400 stats_proxy_->SetMockStats(mock_stats);
2401
2402 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2403 sink_.WaitForEncodedFrame(t);
2404 t += frame_interval_ms;
2405
mflodmancc3d4422017-08-03 08:27:51 -07002406 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002407 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002408 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002409 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2410 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002411 } while (video_source_.sink_wants().max_pixel_count <
2412 last_wants.max_pixel_count ||
2413 video_source_.sink_wants().max_framerate_fps <
2414 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002415
Jonathan Yubc771b72017-12-08 17:04:29 -08002416 // Verify that we've adapted all the way down.
2417 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002418 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002419 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2420 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002421 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002422 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2423 *video_source_.last_sent_height());
2424 EXPECT_EQ(kMinBalancedFramerateFps,
2425 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002426
Jonathan Yubc771b72017-12-08 17:04:29 -08002427 // Adapt back up the same number of times we adapted down.
2428 for (int i = 0; i < loop_count - 1; ++i) {
2429 last_wants = video_source_.sink_wants();
2430
2431 // Simulate the framerate we've been asked to adapt to.
2432 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2433 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2434 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2435 mock_stats.input_frame_rate = fps;
2436 stats_proxy_->SetMockStats(mock_stats);
2437
2438 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2439 sink_.WaitForEncodedFrame(t);
2440 t += frame_interval_ms;
2441
Henrik Boström91aa7322020-04-28 12:24:33 +02002442 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002443 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002444 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002445 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2446 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002447 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2448 last_wants.max_pixel_count ||
2449 video_source_.sink_wants().max_framerate_fps >
2450 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002451 }
2452
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002453 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002454 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002455 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002456 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2457 EXPECT_EQ((loop_count - 1) * 2,
2458 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002459
mflodmancc3d4422017-08-03 08:27:51 -07002460 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002461}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002462
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002463TEST_F(VideoStreamEncoderTest,
2464 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002465 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2466 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002467 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002468
2469 const int kFrameWidth = 1280;
2470 const int kFrameHeight = 720;
2471
2472 int64_t ntp_time = kFrameIntervalMs;
2473
2474 // Force an input frame rate to be available, or the adaptation call won't
2475 // know what framerate to adapt form.
2476 const int kInputFps = 30;
2477 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2478 stats.input_frame_rate = kInputFps;
2479 stats_proxy_->SetMockStats(stats);
2480
2481 video_source_.set_adaptation_enabled(true);
2482 video_stream_encoder_->SetSource(
2483 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002484 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002485 video_source_.IncomingCapturedFrame(
2486 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2487 sink_.WaitForEncodedFrame(ntp_time);
2488 ntp_time += kFrameIntervalMs;
2489
2490 // Trigger CPU overuse.
2491 video_stream_encoder_->TriggerCpuOveruse();
2492 video_source_.IncomingCapturedFrame(
2493 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2494 sink_.WaitForEncodedFrame(ntp_time);
2495 ntp_time += kFrameIntervalMs;
2496
2497 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2498 EXPECT_EQ(std::numeric_limits<int>::max(),
2499 video_source_.sink_wants().max_pixel_count);
2500 // Some framerate constraint should be set.
2501 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2502 EXPECT_LT(restricted_fps, kInputFps);
2503 video_source_.IncomingCapturedFrame(
2504 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2505 sink_.WaitForEncodedFrame(ntp_time);
2506 ntp_time += 100;
2507
Henrik Boström2671dac2020-05-19 16:29:09 +02002508 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002509 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2510 // Give the encoder queue time to process the change in degradation preference
2511 // by waiting for an encoded frame.
2512 video_source_.IncomingCapturedFrame(
2513 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2514 sink_.WaitForEncodedFrame(ntp_time);
2515 ntp_time += kFrameIntervalMs;
2516
2517 video_stream_encoder_->TriggerQualityLow();
2518 video_source_.IncomingCapturedFrame(
2519 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2520 sink_.WaitForEncodedFrame(ntp_time);
2521 ntp_time += kFrameIntervalMs;
2522
2523 // Some resolution constraint should be set.
2524 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2525 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2526 kFrameWidth * kFrameHeight);
2527 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2528
2529 int pixel_count = video_source_.sink_wants().max_pixel_count;
2530 // Triggering a CPU underuse should not change the sink wants since it has
2531 // not been overused for resolution since we changed degradation preference.
2532 video_stream_encoder_->TriggerCpuUnderuse();
2533 video_source_.IncomingCapturedFrame(
2534 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2535 sink_.WaitForEncodedFrame(ntp_time);
2536 ntp_time += kFrameIntervalMs;
2537 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2538 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2539
Evan Shrubsole64469032020-06-11 10:45:29 +02002540 // Change the degradation preference back. CPU underuse should not adapt since
2541 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002542 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002543 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2544 video_source_.IncomingCapturedFrame(
2545 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2546 sink_.WaitForEncodedFrame(ntp_time);
2547 ntp_time += 100;
2548 // Resolution adaptations is gone after changing degradation preference.
2549 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2550 EXPECT_EQ(std::numeric_limits<int>::max(),
2551 video_source_.sink_wants().max_pixel_count);
2552 // The fps adaptation from above is now back.
2553 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2554
2555 // Trigger CPU underuse.
2556 video_stream_encoder_->TriggerCpuUnderuse();
2557 video_source_.IncomingCapturedFrame(
2558 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2559 sink_.WaitForEncodedFrame(ntp_time);
2560 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002561 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2562
2563 // Trigger QP underuse, fps should return to normal.
2564 video_stream_encoder_->TriggerQualityHigh();
2565 video_source_.IncomingCapturedFrame(
2566 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2567 sink_.WaitForEncodedFrame(ntp_time);
2568 ntp_time += kFrameIntervalMs;
2569 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002570
2571 video_stream_encoder_->Stop();
2572}
2573
mflodmancc3d4422017-08-03 08:27:51 -07002574TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002575 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002576 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002577 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002578
sprangc5d62e22017-04-02 23:53:04 -07002579 const int kFrameWidth = 1280;
2580 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002581
Åsa Persson8c1bf952018-09-13 10:42:19 +02002582 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002583
kthelgason5e13d412016-12-01 03:59:51 -08002584 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002585 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002586 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002587 frame_timestamp += kFrameIntervalMs;
2588
perkj803d97f2016-11-01 11:45:46 -07002589 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002590 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002591 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002592 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002593 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002594 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002595
asapersson0944a802017-04-07 00:57:58 -07002596 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002597 // wanted resolution.
2598 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2599 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2600 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002601 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002602
2603 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002604 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002605 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002606 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002607 // Give the encoder queue time to process the change in degradation preference
2608 // by waiting for an encoded frame.
2609 new_video_source.IncomingCapturedFrame(
2610 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2611 sink_.WaitForEncodedFrame(frame_timestamp);
2612 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002613 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002614 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002615
sprangc5d62e22017-04-02 23:53:04 -07002616 // Force an input frame rate to be available, or the adaptation call won't
2617 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002618 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002619 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002620 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002621 stats_proxy_->SetMockStats(stats);
2622
mflodmancc3d4422017-08-03 08:27:51 -07002623 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002624 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002625 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002626 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002627 frame_timestamp += kFrameIntervalMs;
2628
2629 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002630 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002631 EXPECT_EQ(std::numeric_limits<int>::max(),
2632 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002633 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002634
asapersson02465b82017-04-10 01:12:52 -07002635 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002636 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2637 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002638 // Give the encoder queue time to process the change in degradation preference
2639 // by waiting for an encoded frame.
2640 new_video_source.IncomingCapturedFrame(
2641 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2642 sink_.WaitForEncodedFrame(frame_timestamp);
2643 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002644 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002645
mflodmancc3d4422017-08-03 08:27:51 -07002646 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002647 new_video_source.IncomingCapturedFrame(
2648 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002649 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002650 frame_timestamp += kFrameIntervalMs;
2651
2652 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002653 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002654
2655 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002656 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002657 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002658 // Give the encoder queue time to process the change in degradation preference
2659 // by waiting for an encoded frame.
2660 new_video_source.IncomingCapturedFrame(
2661 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2662 sink_.WaitForEncodedFrame(frame_timestamp);
2663 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002664 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2665 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002666 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002667 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002668
2669 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002670 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002671 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002672 // Give the encoder queue time to process the change in degradation preference
2673 // by waiting for an encoded frame.
2674 new_video_source.IncomingCapturedFrame(
2675 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2676 sink_.WaitForEncodedFrame(frame_timestamp);
2677 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002678 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2679 EXPECT_EQ(std::numeric_limits<int>::max(),
2680 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002681 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002682
mflodmancc3d4422017-08-03 08:27:51 -07002683 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002684}
2685
mflodmancc3d4422017-08-03 08:27:51 -07002686TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002688 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002689
asaperssonfab67072017-04-04 05:51:49 -07002690 const int kWidth = 1280;
2691 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002692 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002693 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002694 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2695 EXPECT_FALSE(stats.bw_limited_resolution);
2696 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2697
2698 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002699 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002700 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002701 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002702
2703 stats = stats_proxy_->GetStats();
2704 EXPECT_TRUE(stats.bw_limited_resolution);
2705 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2706
2707 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002709 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002710 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002711
2712 stats = stats_proxy_->GetStats();
2713 EXPECT_FALSE(stats.bw_limited_resolution);
2714 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2715 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2716
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002718}
2719
mflodmancc3d4422017-08-03 08:27:51 -07002720TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002721 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002722 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002723
2724 const int kWidth = 1280;
2725 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002726 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002727 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002728 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2729 EXPECT_FALSE(stats.cpu_limited_resolution);
2730 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2731
2732 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002733 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002734 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002735 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002736
2737 stats = stats_proxy_->GetStats();
2738 EXPECT_TRUE(stats.cpu_limited_resolution);
2739 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2740
2741 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002742 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002743 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002744 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002745
2746 stats = stats_proxy_->GetStats();
2747 EXPECT_FALSE(stats.cpu_limited_resolution);
2748 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002749 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002750
mflodmancc3d4422017-08-03 08:27:51 -07002751 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002752}
2753
mflodmancc3d4422017-08-03 08:27:51 -07002754TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002756 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002757
asaperssonfab67072017-04-04 05:51:49 -07002758 const int kWidth = 1280;
2759 const int kHeight = 720;
2760 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002761 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002762 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002763 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002764 EXPECT_FALSE(stats.cpu_limited_resolution);
2765 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2766
asaperssonfab67072017-04-04 05:51:49 -07002767 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002768 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002769 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002771 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002772 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002773 EXPECT_TRUE(stats.cpu_limited_resolution);
2774 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2775
2776 // Set new source with adaptation still enabled.
2777 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002778 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002779 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002780
asaperssonfab67072017-04-04 05:51:49 -07002781 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002782 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002783 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002784 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002785 EXPECT_TRUE(stats.cpu_limited_resolution);
2786 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2787
2788 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002789 video_stream_encoder_->SetSource(&new_video_source,
2790 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002791
asaperssonfab67072017-04-04 05:51:49 -07002792 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002793 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002794 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002795 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002796 EXPECT_FALSE(stats.cpu_limited_resolution);
2797 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2798
2799 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002800 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002801 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002802
asaperssonfab67072017-04-04 05:51:49 -07002803 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002804 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002805 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002806 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002807 EXPECT_TRUE(stats.cpu_limited_resolution);
2808 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2809
asaperssonfab67072017-04-04 05:51:49 -07002810 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002811 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002812 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002814 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002815 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002816 EXPECT_FALSE(stats.cpu_limited_resolution);
2817 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002818 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002819
mflodmancc3d4422017-08-03 08:27:51 -07002820 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002821}
2822
mflodmancc3d4422017-08-03 08:27:51 -07002823TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002825 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002826
asaperssonfab67072017-04-04 05:51:49 -07002827 const int kWidth = 1280;
2828 const int kHeight = 720;
2829 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002830 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002831 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002832 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002833 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002834 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002835
2836 // Set new source with adaptation still enabled.
2837 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002838 video_stream_encoder_->SetSource(&new_video_source,
2839 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002840
asaperssonfab67072017-04-04 05:51:49 -07002841 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002842 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002843 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002844 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002845 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002846 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002847
asaperssonfab67072017-04-04 05:51:49 -07002848 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002849 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002850 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002851 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002852 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002853 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002854 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002855 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002856
asaperssonfab67072017-04-04 05:51:49 -07002857 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002858 video_stream_encoder_->SetSource(&new_video_source,
2859 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002860
asaperssonfab67072017-04-04 05:51:49 -07002861 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002862 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002863 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002864 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002865 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002866 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002867
asapersson02465b82017-04-10 01:12:52 -07002868 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002870 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002871
asaperssonfab67072017-04-04 05:51:49 -07002872 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002874 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002875 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002876 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002877 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2878 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002879
mflodmancc3d4422017-08-03 08:27:51 -07002880 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002881}
2882
mflodmancc3d4422017-08-03 08:27:51 -07002883TEST_F(VideoStreamEncoderTest,
2884 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002886 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002887
2888 const int kWidth = 1280;
2889 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002890 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002891 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002892 video_source_.IncomingCapturedFrame(
2893 CreateFrame(timestamp_ms, kWidth, kHeight));
2894 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002895 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2897 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2898
2899 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002901 timestamp_ms += kFrameIntervalMs;
2902 video_source_.IncomingCapturedFrame(
2903 CreateFrame(timestamp_ms, kWidth, kHeight));
2904 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002905 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2907 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2908
2909 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002910 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002911 timestamp_ms += kFrameIntervalMs;
2912 video_source_.IncomingCapturedFrame(
2913 CreateFrame(timestamp_ms, kWidth, kHeight));
2914 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002915 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2916 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2917 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2918
Niels Möller4db138e2018-04-19 09:04:13 +02002919 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002920 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002921
2922 VideoEncoderConfig video_encoder_config;
2923 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2924 // Make format different, to force recreation of encoder.
2925 video_encoder_config.video_format.parameters["foo"] = "foo";
2926 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002927 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002928 timestamp_ms += kFrameIntervalMs;
2929 video_source_.IncomingCapturedFrame(
2930 CreateFrame(timestamp_ms, kWidth, kHeight));
2931 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002932 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2933 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2934 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2935
mflodmancc3d4422017-08-03 08:27:51 -07002936 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07002937}
2938
mflodmancc3d4422017-08-03 08:27:51 -07002939TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002940 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02002941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002942 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002943
2944 const int kWidth = 1280;
2945 const int kHeight = 720;
2946 int sequence = 1;
2947
2948 // Enable BALANCED preference, no initial limitation.
2949 test::FrameForwarder source;
2950 video_stream_encoder_->SetSource(&source,
2951 webrtc::DegradationPreference::BALANCED);
2952 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2953 WaitForEncodedFrame(sequence++);
2954 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2955 EXPECT_FALSE(stats.cpu_limited_resolution);
2956 EXPECT_FALSE(stats.cpu_limited_framerate);
2957 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2958
2959 // Trigger CPU overuse, should now adapt down.
2960 video_stream_encoder_->TriggerCpuOveruse();
2961 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2962 WaitForEncodedFrame(sequence++);
2963 stats = stats_proxy_->GetStats();
2964 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2965
2966 // Set new degradation preference should clear restrictions since we changed
2967 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002968 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002969 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2970 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2971 WaitForEncodedFrame(sequence++);
2972 stats = stats_proxy_->GetStats();
2973 EXPECT_FALSE(stats.cpu_limited_resolution);
2974 EXPECT_FALSE(stats.cpu_limited_framerate);
2975 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2976
2977 // Force an input frame rate to be available, or the adaptation call won't
2978 // know what framerate to adapt from.
2979 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2980 mock_stats.input_frame_rate = 30;
2981 stats_proxy_->SetMockStats(mock_stats);
2982 video_stream_encoder_->TriggerCpuOveruse();
2983 stats_proxy_->ResetMockStats();
2984 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2985 WaitForEncodedFrame(sequence++);
2986
2987 // We have now adapted once.
2988 stats = stats_proxy_->GetStats();
2989 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
2990
2991 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02002992 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2993 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01002994 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
2995 WaitForEncodedFrame(sequence++);
2996 stats = stats_proxy_->GetStats();
2997 EXPECT_FALSE(stats.cpu_limited_resolution);
2998 EXPECT_FALSE(stats.cpu_limited_framerate);
2999 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3000
3001 video_stream_encoder_->Stop();
3002}
3003
3004TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003005 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003007 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003008
asapersson0944a802017-04-07 00:57:58 -07003009 const int kWidth = 1280;
3010 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003011 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003012
asaperssonfab67072017-04-04 05:51:49 -07003013 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003014 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003015 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003016 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003017 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003018 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3019
asapersson02465b82017-04-10 01:12:52 -07003020 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003021 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003022 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003023 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003024 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003025 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003026 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003027 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3028
3029 // Set new source with adaptation still enabled.
3030 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003031 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003032 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003033
3034 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003035 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003036 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003037 stats = stats_proxy_->GetStats();
3038 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003039 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003040 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3041
sprangc5d62e22017-04-02 23:53:04 -07003042 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003043 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003044 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003045 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003046 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003047 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003048 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003049 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003050 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003051 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003052 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3053
sprangc5d62e22017-04-02 23:53:04 -07003054 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003055 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003056 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3057 mock_stats.input_frame_rate = 30;
3058 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003059 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003060 stats_proxy_->ResetMockStats();
3061
3062 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003063 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003064 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003065
3066 // Framerate now adapted.
3067 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003068 EXPECT_FALSE(stats.cpu_limited_resolution);
3069 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003070 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3071
3072 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003073 video_stream_encoder_->SetSource(&new_video_source,
3074 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003075 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003076 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003077 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003078
3079 stats = stats_proxy_->GetStats();
3080 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003081 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003082 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3083
3084 // Try to trigger overuse. Should not succeed.
3085 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003086 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003087 stats_proxy_->ResetMockStats();
3088
3089 stats = stats_proxy_->GetStats();
3090 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003091 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003092 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3093
3094 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003095 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003096 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003097 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003098 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003099 stats = stats_proxy_->GetStats();
3100 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003101 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003102 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003103
3104 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003105 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003106 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003107 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003108 stats = stats_proxy_->GetStats();
3109 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003110 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003111 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3112
3113 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003114 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003115 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003116 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003117 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003118 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003119 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003120 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003121 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003122 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003123 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3124
3125 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003126 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003127 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003128 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003129 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003130 stats = stats_proxy_->GetStats();
3131 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003132 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003133 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003134 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003135
mflodmancc3d4422017-08-03 08:27:51 -07003136 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003137}
3138
mflodmancc3d4422017-08-03 08:27:51 -07003139TEST_F(VideoStreamEncoderTest,
3140 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003141 const int kWidth = 1280;
3142 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003143 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003144 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003145
asaperssonfab67072017-04-04 05:51:49 -07003146 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003147 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003148
asaperssonfab67072017-04-04 05:51:49 -07003149 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003150 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003151
asaperssonfab67072017-04-04 05:51:49 -07003152 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003154
asaperssonfab67072017-04-04 05:51:49 -07003155 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003156 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003157
kthelgason876222f2016-11-29 01:44:11 -08003158 // Expect a scale down.
3159 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003160 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003161
asapersson02465b82017-04-10 01:12:52 -07003162 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003163 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003166
asaperssonfab67072017-04-04 05:51:49 -07003167 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003168 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003169 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003170 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003171
asaperssonfab67072017-04-04 05:51:49 -07003172 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003173 EXPECT_EQ(std::numeric_limits<int>::max(),
3174 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003175
asaperssonfab67072017-04-04 05:51:49 -07003176 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003177 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003178 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003179 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003180
asapersson02465b82017-04-10 01:12:52 -07003181 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003182 EXPECT_EQ(std::numeric_limits<int>::max(),
3183 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003184
mflodmancc3d4422017-08-03 08:27:51 -07003185 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003186}
3187
mflodmancc3d4422017-08-03 08:27:51 -07003188TEST_F(VideoStreamEncoderTest,
3189 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003190 const int kWidth = 1280;
3191 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003193 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003194
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003195 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003196 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003197 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003198 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003199
3200 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003201 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003202 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003203 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3204 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3205
3206 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003208 EXPECT_THAT(source.sink_wants(),
3209 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003210 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3211 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3212 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3213
3214 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003216 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3217 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3218 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3219
mflodmancc3d4422017-08-03 08:27:51 -07003220 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003221}
3222
mflodmancc3d4422017-08-03 08:27:51 -07003223TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003224 const int kWidth = 1280;
3225 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003227 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003228
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003229 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003230 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003231 video_stream_encoder_->SetSource(&source,
3232 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003233 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3234 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003235 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003236
3237 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003238 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003239 EXPECT_THAT(source.sink_wants(),
3240 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003241 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3242 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3243 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3244
3245 // Trigger adapt down for same input resolution, expect no change.
3246 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3247 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003248 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003249 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3250 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3251 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3252
3253 // Trigger adapt down for larger input resolution, expect no change.
3254 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3255 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003256 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003257 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3258 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3259 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3260
mflodmancc3d4422017-08-03 08:27:51 -07003261 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003262}
3263
mflodmancc3d4422017-08-03 08:27:51 -07003264TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003265 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3266 const int kWidth = 640;
3267 const int kHeight = 360;
3268 const int64_t kFrameIntervalMs = 150;
3269 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003270 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003271
3272 // Enable BALANCED preference, no initial limitation.
3273 AdaptingFrameForwarder source(&time_controller_);
3274 source.set_adaptation_enabled(true);
3275 video_stream_encoder_->SetSource(&source,
3276 webrtc::DegradationPreference::BALANCED);
3277
3278 int64_t timestamp_ms = kFrameIntervalMs;
3279 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3280 sink_.WaitForEncodedFrame(kWidth, kHeight);
3281 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3282 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3283 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3284 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3285
3286 // Trigger adapt down, expect reduced fps (640x360@15fps).
3287 video_stream_encoder_->TriggerQualityLow();
3288 timestamp_ms += kFrameIntervalMs;
3289 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3290 sink_.WaitForEncodedFrame(timestamp_ms);
3291 EXPECT_THAT(source.sink_wants(),
3292 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3293 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3294 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3295 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3296
3297 // Source requests 270p, expect reduced resolution (480x270@15fps).
3298 source.OnOutputFormatRequest(480, 270);
3299 timestamp_ms += kFrameIntervalMs;
3300 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3301 WaitForEncodedFrame(480, 270);
3302 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3303
3304 // Trigger adapt down, expect reduced fps (480x270@10fps).
3305 video_stream_encoder_->TriggerQualityLow();
3306 timestamp_ms += kFrameIntervalMs;
3307 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3308 sink_.WaitForEncodedFrame(timestamp_ms);
3309 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3312 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3313
3314 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3315 source.OnOutputFormatRequest(320, 180);
3316 timestamp_ms += kFrameIntervalMs;
3317 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3318 WaitForEncodedFrame(320, 180);
3319 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3320
3321 // Trigger adapt down, expect reduced fps (320x180@7fps).
3322 video_stream_encoder_->TriggerQualityLow();
3323 timestamp_ms += kFrameIntervalMs;
3324 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3325 sink_.WaitForEncodedFrame(timestamp_ms);
3326 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3327 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3329 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3330
3331 // Source requests VGA, expect increased resolution (640x360@7fps).
3332 source.OnOutputFormatRequest(640, 360);
3333 timestamp_ms += kFrameIntervalMs;
3334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3335 WaitForEncodedFrame(timestamp_ms);
3336 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3337
3338 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3339 video_stream_encoder_->TriggerQualityHigh();
3340 timestamp_ms += kFrameIntervalMs;
3341 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3342 WaitForEncodedFrame(timestamp_ms);
3343 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3345 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3346 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3347
3348 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3349 video_stream_encoder_->TriggerQualityHigh();
3350 timestamp_ms += kFrameIntervalMs;
3351 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3352 WaitForEncodedFrame(timestamp_ms);
3353 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3354 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3355 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3356 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3357
3358 // Trigger adapt up, expect increased fps (640x360@maxfps).
3359 video_stream_encoder_->TriggerQualityHigh();
3360 timestamp_ms += kFrameIntervalMs;
3361 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3362 WaitForEncodedFrame(timestamp_ms);
3363 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3364 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3366 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3367
3368 video_stream_encoder_->Stop();
3369}
3370
3371TEST_F(VideoStreamEncoderTest,
3372 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3373 const int kWidth = 1280;
3374 const int kHeight = 720;
3375 const int64_t kFrameIntervalMs = 150;
3376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003377 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003378
3379 // Enable BALANCED preference, no initial limitation.
3380 AdaptingFrameForwarder source(&time_controller_);
3381 source.set_adaptation_enabled(true);
3382 video_stream_encoder_->SetSource(&source,
3383 webrtc::DegradationPreference::BALANCED);
3384
3385 int64_t timestamp_ms = kFrameIntervalMs;
3386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3387 sink_.WaitForEncodedFrame(kWidth, kHeight);
3388 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3390 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3391 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3392
3393 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3394 video_stream_encoder_->TriggerQualityLow();
3395 timestamp_ms += kFrameIntervalMs;
3396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3397 sink_.WaitForEncodedFrame(timestamp_ms);
3398 EXPECT_THAT(source.sink_wants(),
3399 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3400 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3403
3404 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3405 video_stream_encoder_->TriggerQualityLow();
3406 timestamp_ms += kFrameIntervalMs;
3407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3408 sink_.WaitForEncodedFrame(timestamp_ms);
3409 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3410 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3411 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3412 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3413
3414 // Trigger adapt down, expect reduced fps (640x360@15fps).
3415 video_stream_encoder_->TriggerQualityLow();
3416 timestamp_ms += kFrameIntervalMs;
3417 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3418 WaitForEncodedFrame(timestamp_ms);
3419 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3420 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3422 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3425 source.OnOutputFormatRequest(320, 180);
3426 timestamp_ms += kFrameIntervalMs;
3427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3428 WaitForEncodedFrame(320, 180);
3429 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3431
3432 // Trigger adapt down, expect reduced fps (320x180@7fps).
3433 video_stream_encoder_->TriggerCpuOveruse();
3434 timestamp_ms += kFrameIntervalMs;
3435 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3436 WaitForEncodedFrame(timestamp_ms);
3437 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3439 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3440 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3441 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3442 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3443 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3444
3445 // Source requests HD, expect increased resolution (640x360@7fps).
3446 source.OnOutputFormatRequest(1280, 720);
3447 timestamp_ms += kFrameIntervalMs;
3448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3449 WaitForEncodedFrame(timestamp_ms);
3450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3451 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3452
3453 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3454 video_stream_encoder_->TriggerCpuUnderuse();
3455 timestamp_ms += kFrameIntervalMs;
3456 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3457 WaitForEncodedFrame(timestamp_ms);
3458 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3459 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3460 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3461 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3462 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3463 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3464 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3465
3466 // Trigger adapt up, expect increased fps (640x360@maxfps).
3467 video_stream_encoder_->TriggerQualityHigh();
3468 video_stream_encoder_->TriggerCpuUnderuse();
3469 timestamp_ms += kFrameIntervalMs;
3470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3471 WaitForEncodedFrame(timestamp_ms);
3472 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3473 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3474 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3475 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3476 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3477 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3479
3480 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3481 video_stream_encoder_->TriggerQualityHigh();
3482 video_stream_encoder_->TriggerCpuUnderuse();
3483 timestamp_ms += kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3485 WaitForEncodedFrame(timestamp_ms);
3486 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3489 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3490 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3491 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3493
3494 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3495 video_stream_encoder_->TriggerQualityHigh();
3496 video_stream_encoder_->TriggerCpuUnderuse();
3497 timestamp_ms += kFrameIntervalMs;
3498 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3499 WaitForEncodedFrame(timestamp_ms);
3500 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3503 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3504 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3505 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3506 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3507
3508 video_stream_encoder_->Stop();
3509}
3510
3511TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003512 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003513 const int kWidth = 1280;
3514 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003516 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003517
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003518 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003519 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003520 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003521 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003522
3523 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003524 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003526 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3527 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3528
3529 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003530 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003531 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3533 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3534
mflodmancc3d4422017-08-03 08:27:51 -07003535 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003536}
3537
mflodmancc3d4422017-08-03 08:27:51 -07003538TEST_F(VideoStreamEncoderTest,
3539 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003540 const int kWidth = 1280;
3541 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003543 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003544
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003545 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003546 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003547 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003548 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003549
3550 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003551 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003553 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003554 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3555
3556 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003557 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003560 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3561
mflodmancc3d4422017-08-03 08:27:51 -07003562 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003563}
3564
mflodmancc3d4422017-08-03 08:27:51 -07003565TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003566 const int kWidth = 1280;
3567 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003569 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003570
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003571 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003572 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003573 video_stream_encoder_->SetSource(&source,
3574 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003575
3576 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3577 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003578 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003579 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3581 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582
3583 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003584 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003585 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3587 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3588 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3589
mflodmancc3d4422017-08-03 08:27:51 -07003590 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003591}
3592
mflodmancc3d4422017-08-03 08:27:51 -07003593TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003594 const int kWidth = 1280;
3595 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003597 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003598
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003599 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003600 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003601 video_stream_encoder_->SetSource(&source,
3602 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003603
3604 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3605 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003606 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003607 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3609 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3610
3611 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003612 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003613 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003614 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3615 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3616 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3617
mflodmancc3d4422017-08-03 08:27:51 -07003618 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003619}
3620
mflodmancc3d4422017-08-03 08:27:51 -07003621TEST_F(VideoStreamEncoderTest,
3622 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003623 const int kWidth = 1280;
3624 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003626 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003627
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003628 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003629 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003630 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003632 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003633
3634 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003635 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003636 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003637 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3638 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3639
3640 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003641 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003642 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003643 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003644 EXPECT_THAT(source.sink_wants(),
3645 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3647 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3648
3649 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003650 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003651 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003652 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3653 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3654 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3655
mflodmancc3d4422017-08-03 08:27:51 -07003656 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003657}
3658
mflodmancc3d4422017-08-03 08:27:51 -07003659TEST_F(VideoStreamEncoderTest,
3660 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003661 const int kWidth = 1280;
3662 const int kHeight = 720;
3663 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003665 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003666
3667 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3668 stats.input_frame_rate = kInputFps;
3669 stats_proxy_->SetMockStats(stats);
3670
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003671 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003672 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3673 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003674 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003675
3676 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003677 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003678 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3679 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003680 EXPECT_THAT(video_source_.sink_wants(),
3681 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003682
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003683 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003684 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003685 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003686 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003687 // Give the encoder queue time to process the change in degradation preference
3688 // by waiting for an encoded frame.
3689 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3690 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003691 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003692
3693 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003694 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003695 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3696 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003697 EXPECT_THAT(new_video_source.sink_wants(),
3698 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003699
3700 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003701 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003702 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003703
mflodmancc3d4422017-08-03 08:27:51 -07003704 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003705}
3706
mflodmancc3d4422017-08-03 08:27:51 -07003707TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003708 const int kWidth = 1280;
3709 const int kHeight = 720;
3710 const size_t kNumFrames = 10;
3711
Henrik Boström381d1092020-05-12 18:49:07 +02003712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003714
asaperssond0de2952017-04-21 01:47:31 -07003715 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003716 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003717 video_source_.set_adaptation_enabled(true);
3718
3719 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3721
3722 int downscales = 0;
3723 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003724 video_source_.IncomingCapturedFrame(
3725 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3726 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003727
asaperssonfab67072017-04-04 05:51:49 -07003728 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003729 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003730 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003731 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003732
3733 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3734 ++downscales;
3735
3736 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3737 EXPECT_EQ(downscales,
3738 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3739 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003740 }
mflodmancc3d4422017-08-03 08:27:51 -07003741 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003742}
3743
mflodmancc3d4422017-08-03 08:27:51 -07003744TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003745 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3746 const int kWidth = 1280;
3747 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003749 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003750
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003751 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003752 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003753 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003755 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003756
Åsa Persson8c1bf952018-09-13 10:42:19 +02003757 int64_t timestamp_ms = kFrameIntervalMs;
3758 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003759 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003761 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3762 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3763
3764 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003765 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003766 timestamp_ms += kFrameIntervalMs;
3767 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3768 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003769 EXPECT_THAT(source.sink_wants(),
3770 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003771 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3772 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3773
3774 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003775 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003776 timestamp_ms += kFrameIntervalMs;
3777 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003778 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003779 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3781 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3782
3783 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003784 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003785 timestamp_ms += kFrameIntervalMs;
3786 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3787 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003788 EXPECT_THAT(source.sink_wants(),
3789 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003790 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3791 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3792
3793 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003794 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003795 timestamp_ms += kFrameIntervalMs;
3796 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003797 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3800 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3801
mflodmancc3d4422017-08-03 08:27:51 -07003802 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003803}
3804
mflodmancc3d4422017-08-03 08:27:51 -07003805TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003806 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3807 const int kWidth = 1280;
3808 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003810 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003811
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003812 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003813 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003814 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003815 video_stream_encoder_->SetSource(&source,
3816 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003817
Åsa Persson8c1bf952018-09-13 10:42:19 +02003818 int64_t timestamp_ms = kFrameIntervalMs;
3819 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003820 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003821 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003822 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3824
3825 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003826 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003827 timestamp_ms += kFrameIntervalMs;
3828 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3829 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003830 EXPECT_THAT(source.sink_wants(),
3831 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003832 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3833 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3834
3835 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003836 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003837 timestamp_ms += kFrameIntervalMs;
3838 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003839 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003840 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003841 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3842 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3843
3844 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003845 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003846 timestamp_ms += kFrameIntervalMs;
3847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3848 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003849 EXPECT_THAT(source.sink_wants(),
3850 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003851 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3852 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3853
3854 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003855 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003856 timestamp_ms += kFrameIntervalMs;
3857 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003858 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003859 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003860 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3861 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3862
mflodmancc3d4422017-08-03 08:27:51 -07003863 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003864}
3865
Sergey Silkin41c650b2019-10-14 13:12:19 +02003866TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3867 fake_encoder_.SetResolutionBitrateLimits(
3868 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3869
Henrik Boström381d1092020-05-12 18:49:07 +02003870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003871 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3872 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3873 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3874 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003875
3876 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003877 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003878 source.set_adaptation_enabled(true);
3879 video_stream_encoder_->SetSource(
3880 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3881
3882 // Insert 720p frame.
3883 int64_t timestamp_ms = kFrameIntervalMs;
3884 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3885 WaitForEncodedFrame(1280, 720);
3886
3887 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003889 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3890 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3891 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3892 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003893 video_stream_encoder_->TriggerQualityLow();
3894
3895 // Insert 720p frame. It should be downscaled and encoded.
3896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3898 WaitForEncodedFrame(960, 540);
3899
3900 // Trigger adapt up. Higher resolution should not be requested duo to lack
3901 // of bitrate.
3902 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003903 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003904
3905 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003906 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003907 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3908 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3909 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3910 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003911
3912 // Trigger adapt up. Higher resolution should be requested.
3913 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003914 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003915
3916 video_stream_encoder_->Stop();
3917}
3918
3919TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3920 fake_encoder_.SetResolutionBitrateLimits(
3921 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3922
3923 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003925 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3926 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3927 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3928 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003929
3930 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003931 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003932 source.set_adaptation_enabled(true);
3933 video_stream_encoder_->SetSource(
3934 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3935
3936 // Insert 720p frame. It should be dropped and lower resolution should be
3937 // requested.
3938 int64_t timestamp_ms = kFrameIntervalMs;
3939 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3940 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003941 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003942
3943 // Insert 720p frame. It should be downscaled and encoded.
3944 timestamp_ms += kFrameIntervalMs;
3945 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3946 WaitForEncodedFrame(960, 540);
3947
3948 video_stream_encoder_->Stop();
3949}
3950
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003951class BalancedDegradationTest : public VideoStreamEncoderTest {
3952 protected:
3953 void SetupTest() {
3954 // Reset encoder for field trials to take effect.
3955 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02003956 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003957
3958 // Enable BALANCED preference.
3959 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003960 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3961 }
3962
Asa Persson606d3cb2021-10-04 10:07:11 +02003963 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02003964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003965 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003966 }
3967
Åsa Persson45b176f2019-09-30 11:19:05 +02003968 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003969 timestamp_ms_ += kFrameIntervalMs;
3970 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003971 }
3972
3973 void InsertFrameAndWaitForEncoded() {
3974 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003975 sink_.WaitForEncodedFrame(timestamp_ms_);
3976 }
3977
3978 const int kWidth = 640; // pixels:640x360=230400
3979 const int kHeight = 360;
3980 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3981 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003982 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003983};
3984
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003985TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003986 test::ScopedFieldTrials field_trials(
3987 "WebRTC-Video-BalancedDegradationSettings/"
3988 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3989 SetupTest();
3990
3991 // Force input frame rate.
3992 const int kInputFps = 24;
3993 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3994 stats.input_frame_rate = kInputFps;
3995 stats_proxy_->SetMockStats(stats);
3996
Åsa Persson45b176f2019-09-30 11:19:05 +02003997 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003998 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003999
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004000 // Trigger adapt down, expect scaled down framerate and resolution,
4001 // since Fps diff (input-requested:0) < threshold.
4002 video_stream_encoder_->TriggerQualityLow();
4003 EXPECT_THAT(source_.sink_wants(),
4004 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004005
4006 video_stream_encoder_->Stop();
4007}
4008
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004009TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004010 test::ScopedFieldTrials field_trials(
4011 "WebRTC-Video-BalancedDegradationSettings/"
4012 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4013 SetupTest();
4014
4015 // Force input frame rate.
4016 const int kInputFps = 25;
4017 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4018 stats.input_frame_rate = kInputFps;
4019 stats_proxy_->SetMockStats(stats);
4020
Åsa Persson45b176f2019-09-30 11:19:05 +02004021 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004022 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004023
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004024 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4025 // Fps diff (input-requested:1) == threshold.
4026 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004027 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004028
4029 video_stream_encoder_->Stop();
4030}
4031
4032TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4033 test::ScopedFieldTrials field_trials(
4034 "WebRTC-Video-BalancedDegradationSettings/"
4035 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4036 SetupTest();
4037
4038 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4039
Åsa Persson45b176f2019-09-30 11:19:05 +02004040 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004041 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004042
4043 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4044 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004045 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004046
4047 video_stream_encoder_->Stop();
4048}
4049
Åsa Perssonccfb3402019-09-25 15:13:04 +02004050TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004051 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004052 "WebRTC-Video-BalancedDegradationSettings/"
4053 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004054 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004055
Asa Persson606d3cb2021-10-04 10:07:11 +02004056 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4057 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4058 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004059
Åsa Persson45b176f2019-09-30 11:19:05 +02004060 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004061 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004062 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4063
4064 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4065 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004066 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004067 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004068 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4069
4070 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4071 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004072 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004073 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004074 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4075
Åsa Persson30ab0152019-08-27 12:22:33 +02004076 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4077 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004078 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004079 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004080 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004081 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4082
4083 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004084 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004085 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004086 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004087
Åsa Persson30ab0152019-08-27 12:22:33 +02004088 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004089 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004090 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004091 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004092 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004093 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4094
4095 video_stream_encoder_->Stop();
4096}
4097
Åsa Perssonccfb3402019-09-25 15:13:04 +02004098TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004099 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4100 test::ScopedFieldTrials field_trials(
4101 "WebRTC-Video-BalancedDegradationSettings/"
4102 "pixels:57600|129600|230400,fps:7|24|24/");
4103 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004104 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004105
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004106 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004107
4108 // Insert frame, expect scaled down:
4109 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4110 InsertFrame();
4111 EXPECT_FALSE(WaitForFrame(1000));
4112 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4113 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4114
4115 // Insert frame, expect scaled down:
4116 // resolution (320x180@24fps).
4117 InsertFrame();
4118 EXPECT_FALSE(WaitForFrame(1000));
4119 EXPECT_LT(source_.sink_wants().max_pixel_count,
4120 source_.last_wants().max_pixel_count);
4121 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4122
4123 // Frame should not be dropped (min pixels per frame reached).
4124 InsertFrameAndWaitForEncoded();
4125
4126 video_stream_encoder_->Stop();
4127}
4128
4129TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004130 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004131 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004132 "WebRTC-Video-BalancedDegradationSettings/"
4133 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004134 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004135
Asa Persson606d3cb2021-10-04 10:07:11 +02004136 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4137 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4138 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004139
Åsa Persson45b176f2019-09-30 11:19:05 +02004140 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004141 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004142 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4143
4144 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4145 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004146 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004147 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004148 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4149
4150 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4151 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004152 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004153 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004154 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4155
4156 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4157 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004158 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004159 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004160 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4161
Åsa Persson30ab0152019-08-27 12:22:33 +02004162 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4163 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004164 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004165 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004166 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4167
4168 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4169 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004170 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004171 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4172
4173 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004174 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004175 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004176 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004177 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004178 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4179
4180 video_stream_encoder_->Stop();
4181}
4182
Åsa Perssonccfb3402019-09-25 15:13:04 +02004183TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004184 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004185 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004186 "WebRTC-Video-BalancedDegradationSettings/"
4187 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004188 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004189
Asa Persson606d3cb2021-10-04 10:07:11 +02004190 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4191 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4192 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4193 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4194 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004195
Åsa Persson45b176f2019-09-30 11:19:05 +02004196 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004197 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004198 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4199
4200 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4201 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004202 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004203 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004204 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4207 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004209 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004210 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4211
4212 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004215 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004216 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4217
4218 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4219 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004220 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004221 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4222
4223 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004224 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004225 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004226 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004227 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004228 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4229
4230 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004231 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004232 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004233 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004234 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4235
4236 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004237 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004238 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004239 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004240 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004241 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4242
Åsa Persson1b247f12019-08-14 17:26:39 +02004243 video_stream_encoder_->Stop();
4244}
4245
mflodmancc3d4422017-08-03 08:27:51 -07004246TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004247 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4248 const int kWidth = 1280;
4249 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004250 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004251 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004252
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004253 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004254 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004255 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004256 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004257 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004258
Åsa Persson8c1bf952018-09-13 10:42:19 +02004259 int64_t timestamp_ms = kFrameIntervalMs;
4260 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004261 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004262 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004263 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4264 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4265 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4266 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4267
4268 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004269 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004270 timestamp_ms += kFrameIntervalMs;
4271 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4272 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004273 EXPECT_THAT(source.sink_wants(),
4274 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004275 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4276 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4277 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4278 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4279
4280 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004281 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004282 timestamp_ms += kFrameIntervalMs;
4283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4284 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004285 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004286 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4288 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4289 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4290
Jonathan Yubc771b72017-12-08 17:04:29 -08004291 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004292 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004293 timestamp_ms += kFrameIntervalMs;
4294 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4295 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004296 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004297 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004299 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004300 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4301
Jonathan Yubc771b72017-12-08 17:04:29 -08004302 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004303 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004304 timestamp_ms += kFrameIntervalMs;
4305 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4306 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004307 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004308 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004309 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4310 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4311 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4312 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
Jonathan Yubc771b72017-12-08 17:04:29 -08004314 // Trigger quality adapt down, expect no change (min resolution reached).
4315 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004316 timestamp_ms += kFrameIntervalMs;
4317 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4318 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004319 EXPECT_THAT(source.sink_wants(), FpsMax());
4320 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004321 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4322 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4323 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
Evan Shrubsole64469032020-06-11 10:45:29 +02004326 // Trigger quality adapt up, expect upscaled resolution (480x270).
4327 video_stream_encoder_->TriggerQualityHigh();
4328 timestamp_ms += kFrameIntervalMs;
4329 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4330 WaitForEncodedFrame(timestamp_ms);
4331 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4332 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4333 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4334 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4335 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger quality and cpu adapt up since both are most limited, expect
4338 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004339 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004340 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004341 timestamp_ms += kFrameIntervalMs;
4342 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4343 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004344 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004345 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4346 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4347 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004348 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004349
Evan Shrubsole64469032020-06-11 10:45:29 +02004350 // Trigger quality and cpu adapt up since both are most limited, expect
4351 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004352 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004354 timestamp_ms += kFrameIntervalMs;
4355 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4356 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004357 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004358 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004359 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004360 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004361 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4362 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004363
Evan Shrubsole64469032020-06-11 10:45:29 +02004364 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4365 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004366 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004367 timestamp_ms += kFrameIntervalMs;
4368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4369 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004370 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004371 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004373 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004374 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004375
4376 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004377 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004378 timestamp_ms += kFrameIntervalMs;
4379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004380 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004381 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004382 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004383 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004385 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004386 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004387
mflodmancc3d4422017-08-03 08:27:51 -07004388 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004389}
4390
mflodmancc3d4422017-08-03 08:27:51 -07004391TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004392 const int kWidth = 640;
4393 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004394
Henrik Boström381d1092020-05-12 18:49:07 +02004395 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004396 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004397
perkj803d97f2016-11-01 11:45:46 -07004398 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004399 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004400 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004401 }
4402
mflodmancc3d4422017-08-03 08:27:51 -07004403 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004404 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004405 video_source_.IncomingCapturedFrame(CreateFrame(
4406 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004407 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004408 }
4409
mflodmancc3d4422017-08-03 08:27:51 -07004410 video_stream_encoder_->Stop();
4411 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004412 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004413
Ying Wangef3998f2019-12-09 13:06:53 +01004414 EXPECT_METRIC_EQ(
4415 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4416 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004417 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4418}
4419
mflodmancc3d4422017-08-03 08:27:51 -07004420TEST_F(VideoStreamEncoderTest,
4421 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004423 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004424 const int kWidth = 640;
4425 const int kHeight = 360;
4426
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004427 video_stream_encoder_->SetSource(&video_source_,
4428 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004429
4430 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4431 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004432 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004433 }
4434
mflodmancc3d4422017-08-03 08:27:51 -07004435 video_stream_encoder_->Stop();
4436 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004437 stats_proxy_.reset();
4438
4439 EXPECT_EQ(0,
4440 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4441}
4442
Per Kjellanderdcef6412020-10-07 15:09:05 +02004443TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4444 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004445 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004446 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004447
4448 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004449 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004450 SimulcastRateAllocator(fake_encoder_.config())
4451 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004452 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004453
Henrik Boström381d1092020-05-12 18:49:07 +02004454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004455 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004456
sprang57c2fff2017-01-16 06:24:02 -08004457 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004458 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4459 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004460 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4461 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4462
Erik Språngd7329ca2019-02-21 21:19:53 +01004463 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004464 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004465 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004466
Per Kjellanderdcef6412020-10-07 15:09:05 +02004467 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004468 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004469 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4470 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004471 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004472 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004473
Per Kjellanderdcef6412020-10-07 15:09:05 +02004474 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004475 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004476 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004477 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004478 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4479 WaitForEncodedFrame(CurrentTimeMs());
4480 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004481 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004482 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004483
mflodmancc3d4422017-08-03 08:27:51 -07004484 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004485}
4486
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004487TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004488 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004489 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004490 kVideoLayersAllocation);
4491
4492 const int kDefaultFps = 30;
4493
4494 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004495 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004496
4497 video_source_.IncomingCapturedFrame(
4498 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4499 WaitForEncodedFrame(CurrentTimeMs());
4500 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4501 VideoLayersAllocation last_layer_allocation =
4502 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004503 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004504 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4505
4506 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004507 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004508 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004509 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004510 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4511
Erik Språng9d69cbe2020-10-22 17:44:42 +02004512 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004513 int number_of_layers_allocation = 1;
4514 const int64_t start_time_ms = CurrentTimeMs();
4515 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4516 video_source_.IncomingCapturedFrame(
4517 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4518 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004519 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4520 number_of_layers_allocation = sink_.number_of_layers_allocations();
4521 VideoLayersAllocation new_allocation =
4522 sink_.GetLastVideoLayersAllocation();
4523 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4524 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4525 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4526 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4527 .target_bitrate_per_temporal_layer,
4528 last_layer_allocation.active_spatial_layers[0]
4529 .target_bitrate_per_temporal_layer);
4530 last_layer_allocation = new_allocation;
4531 }
4532 }
4533 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4534 video_stream_encoder_->Stop();
4535}
4536
4537TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004538 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004539 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4540 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4541 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004542 VideoEncoderConfig video_encoder_config;
4543 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4544 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004545 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004546 video_encoder_config.content_type =
4547 VideoEncoderConfig::ContentType::kRealtimeVideo;
4548 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004549 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004550 VideoEncoder::GetDefaultVp8Settings());
4551 for (auto& layer : video_encoder_config.simulcast_layers) {
4552 layer.num_temporal_layers = 2;
4553 }
4554 // Simulcast layers are used for enabling/disabling streams.
4555 video_encoder_config.simulcast_layers[0].active = true;
4556 video_encoder_config.simulcast_layers[1].active = false;
4557 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004558 ConfigureEncoder(std::move(video_encoder_config),
4559 VideoStreamEncoder::BitrateAllocationCallbackType::
4560 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004561
4562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004563 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004564
4565 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4566 WaitForEncodedFrame(CurrentTimeMs());
4567 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4568 VideoLayersAllocation last_layer_allocation =
4569 sink_.GetLastVideoLayersAllocation();
4570
4571 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4572 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4573 .target_bitrate_per_temporal_layer,
4574 SizeIs(2));
4575 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4576 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4577 video_stream_encoder_->Stop();
4578}
4579
4580TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004581 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004582 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4583 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4584 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004585 VideoEncoderConfig video_encoder_config;
4586 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4587 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004588 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004589 video_encoder_config.content_type =
4590 VideoEncoderConfig::ContentType::kRealtimeVideo;
4591 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004592 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004593 VideoEncoder::GetDefaultVp8Settings());
4594 for (auto& layer : video_encoder_config.simulcast_layers) {
4595 layer.num_temporal_layers = 2;
4596 }
4597 // Simulcast layers are used for enabling/disabling streams.
4598 video_encoder_config.simulcast_layers[0].active = true;
4599 video_encoder_config.simulcast_layers[1].active = false;
4600 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004601 ConfigureEncoder(std::move(video_encoder_config),
4602 VideoStreamEncoder::BitrateAllocationCallbackType::
4603 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004604
4605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004607
4608 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4609 WaitForEncodedFrame(CurrentTimeMs());
4610 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4611 VideoLayersAllocation last_layer_allocation =
4612 sink_.GetLastVideoLayersAllocation();
4613
4614 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4615 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4616 .target_bitrate_per_temporal_layer,
4617 SizeIs(2));
4618 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4619
4620 video_stream_encoder_->Stop();
4621}
4622
4623TEST_F(VideoStreamEncoderTest,
4624 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4625 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4626 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004627 VideoEncoderConfig video_encoder_config;
4628 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4629 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004630 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004631 video_encoder_config.content_type =
4632 VideoEncoderConfig::ContentType::kRealtimeVideo;
4633 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4634 vp9_settings.numberOfSpatialLayers = 2;
4635 vp9_settings.numberOfTemporalLayers = 2;
4636 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4637 vp9_settings.automaticResizeOn = false;
4638 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004639 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004640 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004641 ConfigureEncoder(std::move(video_encoder_config),
4642 VideoStreamEncoder::BitrateAllocationCallbackType::
4643 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004644
4645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004646 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004647
4648 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4649 WaitForEncodedFrame(CurrentTimeMs());
4650 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4651 VideoLayersAllocation last_layer_allocation =
4652 sink_.GetLastVideoLayersAllocation();
4653
4654 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4655 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4656 .target_bitrate_per_temporal_layer,
4657 SizeIs(2));
4658 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4659 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4660 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4661 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4662 .target_bitrate_per_temporal_layer,
4663 SizeIs(2));
4664 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4665 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4666 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4667
4668 // Since full SVC is used, expect the top layer to utilize the full target
4669 // rate.
4670 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4671 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004672 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004673 video_stream_encoder_->Stop();
4674}
4675
4676TEST_F(VideoStreamEncoderTest,
4677 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4678 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4679 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004680 VideoEncoderConfig video_encoder_config;
4681 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4682 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004683 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004684 video_encoder_config.content_type =
4685 VideoEncoderConfig::ContentType::kRealtimeVideo;
4686 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4687 vp9_settings.numberOfSpatialLayers = 2;
4688 vp9_settings.numberOfTemporalLayers = 2;
4689 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4690 vp9_settings.automaticResizeOn = false;
4691 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004692 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004693 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004694 ConfigureEncoder(std::move(video_encoder_config),
4695 VideoStreamEncoder::BitrateAllocationCallbackType::
4696 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004697
4698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004700
4701 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4702 WaitForEncodedFrame(CurrentTimeMs());
4703 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4704 VideoLayersAllocation last_layer_allocation =
4705 sink_.GetLastVideoLayersAllocation();
4706
4707 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4708 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4709 .target_bitrate_per_temporal_layer,
4710 SizeIs(1));
4711 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4712 .target_bitrate_per_temporal_layer,
4713 SizeIs(1));
4714 // Since full SVC is used, expect the top layer to utilize the full target
4715 // rate.
4716 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4717 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004718 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004719 video_stream_encoder_->Stop();
4720}
4721
4722TEST_F(VideoStreamEncoderTest,
4723 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4724 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4725 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004726 VideoEncoderConfig video_encoder_config;
4727 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4728 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004729 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004730 video_encoder_config.content_type =
4731 VideoEncoderConfig::ContentType::kRealtimeVideo;
4732 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4733 vp9_settings.numberOfSpatialLayers = 2;
4734 vp9_settings.numberOfTemporalLayers = 2;
4735 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4736 vp9_settings.automaticResizeOn = false;
4737 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004738 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004739 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004740 ConfigureEncoder(std::move(video_encoder_config),
4741 VideoStreamEncoder::BitrateAllocationCallbackType::
4742 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004743
4744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004745 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004746
4747 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4748 WaitForEncodedFrame(CurrentTimeMs());
4749 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4750 VideoLayersAllocation last_layer_allocation =
4751 sink_.GetLastVideoLayersAllocation();
4752
4753 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4754 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4755 .target_bitrate_per_temporal_layer,
4756 SizeIs(2));
4757 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4758 .target_bitrate_per_temporal_layer,
4759 SizeIs(2));
4760 // Since KSVC is, spatial layers are independend except on key frames.
4761 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4762 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004763 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 video_stream_encoder_->Stop();
4765}
4766
4767TEST_F(VideoStreamEncoderTest,
4768 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4769 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4770 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4771 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 VideoEncoderConfig video_encoder_config;
4773 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4774 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004775 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004776 video_encoder_config.content_type =
4777 VideoEncoderConfig::ContentType::kRealtimeVideo;
4778 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4779 vp9_settings.numberOfSpatialLayers = 3;
4780 vp9_settings.numberOfTemporalLayers = 2;
4781 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4782 vp9_settings.automaticResizeOn = false;
4783 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004784 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004785 vp9_settings);
4786 // Simulcast layers are used for enabling/disabling streams.
4787 video_encoder_config.simulcast_layers.resize(3);
4788 video_encoder_config.simulcast_layers[0].active = false;
4789 video_encoder_config.simulcast_layers[1].active = true;
4790 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004791 ConfigureEncoder(std::move(video_encoder_config),
4792 VideoStreamEncoder::BitrateAllocationCallbackType::
4793 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004794
4795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004797
4798 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4799 WaitForEncodedFrame(CurrentTimeMs());
4800 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4801 VideoLayersAllocation last_layer_allocation =
4802 sink_.GetLastVideoLayersAllocation();
4803
4804 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4805 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4806 .target_bitrate_per_temporal_layer,
4807 SizeIs(2));
4808 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4809 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4810
4811 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4812 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4813 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4814 .target_bitrate_per_temporal_layer,
4815 SizeIs(2));
4816 // Since full SVC is used, expect the top layer to utilize the full target
4817 // rate.
4818 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4819 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004820 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821 video_stream_encoder_->Stop();
4822}
4823
4824TEST_F(VideoStreamEncoderTest,
4825 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4826 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4827 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4828 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004829 VideoEncoderConfig video_encoder_config;
4830 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4831 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004832 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004833 video_encoder_config.content_type =
4834 VideoEncoderConfig::ContentType::kRealtimeVideo;
4835 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4836 vp9_settings.numberOfSpatialLayers = 3;
4837 vp9_settings.numberOfTemporalLayers = 2;
4838 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4839 vp9_settings.automaticResizeOn = false;
4840 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004841 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004842 vp9_settings);
4843 // Simulcast layers are used for enabling/disabling streams.
4844 video_encoder_config.simulcast_layers.resize(3);
4845 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004846 ConfigureEncoder(std::move(video_encoder_config),
4847 VideoStreamEncoder::BitrateAllocationCallbackType::
4848 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004849
4850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004851 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004852
4853 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4854 WaitForEncodedFrame(CurrentTimeMs());
4855 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4856 VideoLayersAllocation last_layer_allocation =
4857 sink_.GetLastVideoLayersAllocation();
4858
4859 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4860 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4861 .target_bitrate_per_temporal_layer,
4862 SizeIs(2));
4863 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4864 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4865
4866 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4867 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4868 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4869 .target_bitrate_per_temporal_layer,
4870 SizeIs(2));
4871 video_stream_encoder_->Stop();
4872}
4873
4874TEST_F(VideoStreamEncoderTest,
4875 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4876 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4877 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4878 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004879 VideoEncoderConfig video_encoder_config;
4880 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4881 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004882 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004883 video_encoder_config.content_type =
4884 VideoEncoderConfig::ContentType::kRealtimeVideo;
4885 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4886 vp9_settings.numberOfSpatialLayers = 3;
4887 vp9_settings.numberOfTemporalLayers = 2;
4888 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4889 vp9_settings.automaticResizeOn = false;
4890 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004891 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004892 vp9_settings);
4893 // Simulcast layers are used for enabling/disabling streams.
4894 video_encoder_config.simulcast_layers.resize(3);
4895 video_encoder_config.simulcast_layers[0].active = false;
4896 video_encoder_config.simulcast_layers[1].active = false;
4897 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004898 ConfigureEncoder(std::move(video_encoder_config),
4899 VideoStreamEncoder::BitrateAllocationCallbackType::
4900 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004901
4902 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004903 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004904
4905 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4906 WaitForEncodedFrame(CurrentTimeMs());
4907 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4908 VideoLayersAllocation last_layer_allocation =
4909 sink_.GetLastVideoLayersAllocation();
4910
4911 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4912 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4913 .target_bitrate_per_temporal_layer,
4914 SizeIs(2));
4915 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4916 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4917 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4918 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004919 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920 video_stream_encoder_->Stop();
4921}
4922
4923TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4924 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004925 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004926 kVideoLayersAllocation);
4927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004929
4930 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4931 WaitForEncodedFrame(CurrentTimeMs());
4932 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4933 VideoLayersAllocation last_layer_allocation =
4934 sink_.GetLastVideoLayersAllocation();
4935
4936 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4937 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4938 .target_bitrate_per_temporal_layer,
4939 SizeIs(1));
4940 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4941 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004942 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004943 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4946 video_stream_encoder_->Stop();
4947}
4948
4949TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004950 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4951 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004952 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004953 kVideoLayersAllocation);
4954
4955 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004956 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004957
4958 video_source_.IncomingCapturedFrame(
4959 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4960 WaitForEncodedFrame(CurrentTimeMs());
4961 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4962 VideoLayersAllocation last_layer_allocation =
4963 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004964 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004965 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4966 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4967 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004968 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02004969
4970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004971 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
4972 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004973 video_source_.IncomingCapturedFrame(
4974 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4975 WaitForEncodedFrame(CurrentTimeMs());
4976
4977 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4978 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4979 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4980 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4981 .target_bitrate_per_temporal_layer[0],
4982 DataRate::Zero());
4983
4984 video_stream_encoder_->Stop();
4985}
4986
Per Kjellander4190ce92020-12-15 17:24:55 +01004987TEST_F(VideoStreamEncoderTest,
4988 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4989 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004990 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004991 kVideoLayersAllocation);
4992
4993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004994 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
4995 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01004996
4997 video_source_.IncomingCapturedFrame(
4998 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4999 WaitForEncodedFrame(CurrentTimeMs());
5000 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5001 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5002 SizeIs(2));
5003 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5004 codec_width_);
5005 EXPECT_EQ(
5006 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5007 codec_height_);
5008
5009 video_source_.IncomingCapturedFrame(
5010 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5011 WaitForEncodedFrame(CurrentTimeMs());
5012 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5013 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5014 SizeIs(2));
5015 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5016 codec_width_ / 2);
5017 EXPECT_EQ(
5018 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5019 codec_height_ / 2);
5020
5021 video_stream_encoder_->Stop();
5022}
5023
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005024TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5025 // 2 TLs configured, temporal layers supported by encoder.
5026 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005027 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005028 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005029 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005030 fake_encoder_.SetTemporalLayersSupported(0, true);
5031
5032 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005033 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005034 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005035 kNumTemporalLayers, /*temporal_id*/ 0,
5036 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005037 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005038 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005039 kNumTemporalLayers, /*temporal_id*/ 1,
5040 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005041 VideoBitrateAllocation expected_bitrate;
5042 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5043 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5044
5045 VerifyAllocatedBitrate(expected_bitrate);
5046 video_stream_encoder_->Stop();
5047}
5048
5049TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5050 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005051 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005052 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005053 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005054 fake_encoder_.SetTemporalLayersSupported(0, false);
5055
5056 // Temporal layers not supported by the encoder.
5057 // Total bitrate should be at ti:0.
5058 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005059 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005060
5061 VerifyAllocatedBitrate(expected_bitrate);
5062 video_stream_encoder_->Stop();
5063}
5064
5065TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005066 webrtc::test::ScopedFieldTrials field_trials(
5067 "WebRTC-Video-QualityScalerSettings/"
5068 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5069 // Reset encoder for field trials to take effect.
5070 ConfigureEncoder(video_encoder_config_.Copy());
5071
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005072 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005073 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005074 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005075 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005076 fake_encoder_.SetTemporalLayersSupported(0, true);
5077 fake_encoder_.SetTemporalLayersSupported(1, false);
5078
5079 const int kS0Bps = 150000;
5080 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005081 kS0Bps *
5082 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5083 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005084 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005085 kS0Bps *
5086 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5087 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005088 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005089 // Temporal layers not supported by si:1.
5090 VideoBitrateAllocation expected_bitrate;
5091 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5092 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5093 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5094
5095 VerifyAllocatedBitrate(expected_bitrate);
5096 video_stream_encoder_->Stop();
5097}
5098
Niels Möller7dc26b72017-12-06 10:27:48 +01005099TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5100 const int kFrameWidth = 1280;
5101 const int kFrameHeight = 720;
5102 const int kFramerate = 24;
5103
Henrik Boström381d1092020-05-12 18:49:07 +02005104 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005105 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005106 test::FrameForwarder source;
5107 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005108 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005109
5110 // Insert a single frame, triggering initial configuration.
5111 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5112 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5113
5114 EXPECT_EQ(
5115 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5116 kDefaultFramerate);
5117
5118 // Trigger reconfigure encoder (without resetting the entire instance).
5119 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005120 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5121 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005122 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005123 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005124 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005125 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5126
5127 // Detector should be updated with fps limit from codec config.
5128 EXPECT_EQ(
5129 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5130 kFramerate);
5131
5132 // Trigger overuse, max framerate should be reduced.
5133 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5134 stats.input_frame_rate = kFramerate;
5135 stats_proxy_->SetMockStats(stats);
5136 video_stream_encoder_->TriggerCpuOveruse();
5137 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5138 int adapted_framerate =
5139 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5140 EXPECT_LT(adapted_framerate, kFramerate);
5141
5142 // Trigger underuse, max framerate should go back to codec configured fps.
5143 // Set extra low fps, to make sure it's actually reset, not just incremented.
5144 stats = stats_proxy_->GetStats();
5145 stats.input_frame_rate = adapted_framerate / 2;
5146 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005147 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005148 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5149 EXPECT_EQ(
5150 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5151 kFramerate);
5152
5153 video_stream_encoder_->Stop();
5154}
5155
5156TEST_F(VideoStreamEncoderTest,
5157 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5158 const int kFrameWidth = 1280;
5159 const int kFrameHeight = 720;
5160 const int kLowFramerate = 15;
5161 const int kHighFramerate = 25;
5162
Henrik Boström381d1092020-05-12 18:49:07 +02005163 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005164 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005165 test::FrameForwarder source;
5166 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005167 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005168
5169 // Trigger initial configuration.
5170 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005171 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5172 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005173 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005174 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005175 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005176 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005177 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5178
5179 EXPECT_EQ(
5180 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5181 kLowFramerate);
5182
5183 // Trigger overuse, max framerate should be reduced.
5184 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5185 stats.input_frame_rate = kLowFramerate;
5186 stats_proxy_->SetMockStats(stats);
5187 video_stream_encoder_->TriggerCpuOveruse();
5188 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5189 int adapted_framerate =
5190 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5191 EXPECT_LT(adapted_framerate, kLowFramerate);
5192
5193 // Reconfigure the encoder with a new (higher max framerate), max fps should
5194 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005195 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005196 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5197 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005198 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005199 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5200
5201 EXPECT_EQ(
5202 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5203 adapted_framerate);
5204
5205 // Trigger underuse, max framerate should go back to codec configured fps.
5206 stats = stats_proxy_->GetStats();
5207 stats.input_frame_rate = adapted_framerate;
5208 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005209 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005210 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5211 EXPECT_EQ(
5212 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5213 kHighFramerate);
5214
5215 video_stream_encoder_->Stop();
5216}
5217
mflodmancc3d4422017-08-03 08:27:51 -07005218TEST_F(VideoStreamEncoderTest,
5219 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005220 const int kFrameWidth = 1280;
5221 const int kFrameHeight = 720;
5222 const int kFramerate = 24;
5223
Henrik Boström381d1092020-05-12 18:49:07 +02005224 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005225 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005226 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005227 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005228 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005229
5230 // Trigger initial configuration.
5231 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005232 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5233 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005234 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005235 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005236 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005237 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005238 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005239
Niels Möller7dc26b72017-12-06 10:27:48 +01005240 EXPECT_EQ(
5241 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5242 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005243
5244 // Trigger overuse, max framerate should be reduced.
5245 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5246 stats.input_frame_rate = kFramerate;
5247 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005248 video_stream_encoder_->TriggerCpuOveruse();
5249 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005250 int adapted_framerate =
5251 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005252 EXPECT_LT(adapted_framerate, kFramerate);
5253
5254 // Change degradation preference to not enable framerate scaling. Target
5255 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005256 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005257 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005258 EXPECT_EQ(
5259 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5260 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005261
mflodmancc3d4422017-08-03 08:27:51 -07005262 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005263}
5264
mflodmancc3d4422017-08-03 08:27:51 -07005265TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005266 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005268 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5269 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5270 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005271 const int kWidth = 640;
5272 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005273
asaperssonfab67072017-04-04 05:51:49 -07005274 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005275
5276 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005277 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005278
5279 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005280 EXPECT_TRUE_WAIT(
5281 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005282
sprangc5d62e22017-04-02 23:53:04 -07005283 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005284
asaperssonfab67072017-04-04 05:51:49 -07005285 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005286 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005287 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005288
5289 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005290 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005291
Henrik Boström2671dac2020-05-19 16:29:09 +02005292 EXPECT_TRUE_WAIT(
5293 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005294
mflodmancc3d4422017-08-03 08:27:51 -07005295 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005296}
5297
mflodmancc3d4422017-08-03 08:27:51 -07005298TEST_F(VideoStreamEncoderTest,
5299 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005300 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005302 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5303 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5304 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005305 const int kWidth = 640;
5306 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005307
5308 // We expect the n initial frames to get dropped.
5309 int i;
5310 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005311 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005312 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005313 }
5314 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005315 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005316 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005317
5318 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005319 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005320
mflodmancc3d4422017-08-03 08:27:51 -07005321 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005322}
5323
mflodmancc3d4422017-08-03 08:27:51 -07005324TEST_F(VideoStreamEncoderTest,
5325 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005326 const int kWidth = 640;
5327 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005329 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005330
5331 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005332 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005333 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005334
asaperssonfab67072017-04-04 05:51:49 -07005335 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005336 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005337 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005338
mflodmancc3d4422017-08-03 08:27:51 -07005339 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005340}
5341
mflodmancc3d4422017-08-03 08:27:51 -07005342TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005343 const int kWidth = 640;
5344 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005345 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005346
5347 VideoEncoderConfig video_encoder_config;
5348 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5349 // Make format different, to force recreation of encoder.
5350 video_encoder_config.video_format.parameters["foo"] = "foo";
5351 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005352 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005354 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005355
kthelgasonb83797b2017-02-14 11:57:25 -08005356 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005357 video_stream_encoder_->SetSource(&video_source_,
5358 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005359
asaperssonfab67072017-04-04 05:51:49 -07005360 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005361 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005362 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005363
mflodmancc3d4422017-08-03 08:27:51 -07005364 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005365 fake_encoder_.SetQualityScaling(true);
5366}
5367
Åsa Persson139f4dc2019-08-02 09:29:58 +02005368TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5369 webrtc::test::ScopedFieldTrials field_trials(
5370 "WebRTC-Video-QualityScalerSettings/"
5371 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5372 // Reset encoder for field trials to take effect.
5373 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005374 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5375 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005376 const int kWidth = 640;
5377 const int kHeight = 360;
5378
Henrik Boström381d1092020-05-12 18:49:07 +02005379 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005380 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005381 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5382 // Frame should not be dropped.
5383 WaitForEncodedFrame(1);
5384
Henrik Boström381d1092020-05-12 18:49:07 +02005385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005386 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5387 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5388 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005389 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5390 // Frame should not be dropped.
5391 WaitForEncodedFrame(2);
5392
Henrik Boström381d1092020-05-12 18:49:07 +02005393 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005394 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5395 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5396 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005397 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5398 // Expect to drop this frame, the wait should time out.
5399 ExpectDroppedFrame();
5400
5401 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005402 EXPECT_TRUE_WAIT(
5403 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005404 video_stream_encoder_->Stop();
5405}
5406
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005407TEST_F(VideoStreamEncoderTest,
5408 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5409 webrtc::test::ScopedFieldTrials field_trials(
5410 "WebRTC-Video-QualityScalerSettings/"
5411 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5412 fake_encoder_.SetQualityScaling(false);
5413 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005414 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5415 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005416 const int kWidth = 640;
5417 const int kHeight = 360;
5418
5419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005420 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005421 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5422 // Frame should not be dropped.
5423 WaitForEncodedFrame(1);
5424
5425 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5426 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5427 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5428 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5429 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5430 // Frame should not be dropped.
5431 WaitForEncodedFrame(2);
5432
5433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5434 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5435 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5436 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5437 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5438 // Not dropped since quality scaling is disabled.
5439 WaitForEncodedFrame(3);
5440
5441 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005442 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005443 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5444
5445 video_stream_encoder_->Stop();
5446}
5447
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005448TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005449 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005450 // Set simulcast.
5451 ResetEncoder("VP8", 3, 1, 1, false);
5452 fake_encoder_.SetQualityScaling(true);
5453 const int kWidth = 1280;
5454 const int kHeight = 720;
5455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005456 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005457 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5458 // Frame should not be dropped.
5459 WaitForEncodedFrame(1);
5460
5461 // Trigger QVGA "singlecast"
5462 // Update the config.
5463 VideoEncoderConfig video_encoder_config;
5464 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5465 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005466 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005467 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005468 "VP8", /*max qp*/ 56, /*screencast*/ false,
5469 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005470 for (auto& layer : video_encoder_config.simulcast_layers) {
5471 layer.num_temporal_layers = 1;
5472 layer.max_framerate = kDefaultFramerate;
5473 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005474 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005475 video_encoder_config.content_type =
5476 VideoEncoderConfig::ContentType::kRealtimeVideo;
5477
5478 video_encoder_config.simulcast_layers[0].active = true;
5479 video_encoder_config.simulcast_layers[1].active = false;
5480 video_encoder_config.simulcast_layers[2].active = false;
5481
5482 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5483 kMaxPayloadLength);
5484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5485
5486 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5487 // Frame should not be dropped.
5488 WaitForEncodedFrame(2);
5489
5490 // Trigger HD "singlecast"
5491 video_encoder_config.simulcast_layers[0].active = false;
5492 video_encoder_config.simulcast_layers[1].active = false;
5493 video_encoder_config.simulcast_layers[2].active = true;
5494
5495 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5496 kMaxPayloadLength);
5497 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5498
5499 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5500 // Frame should be dropped because of initial frame drop.
5501 ExpectDroppedFrame();
5502
5503 // Expect the sink_wants to specify a scaled frame.
5504 EXPECT_TRUE_WAIT(
5505 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5506 video_stream_encoder_->Stop();
5507}
5508
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005509TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005510 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005511 // Set simulcast.
5512 ResetEncoder("VP9", 1, 1, 3, false);
5513 fake_encoder_.SetQualityScaling(true);
5514 const int kWidth = 1280;
5515 const int kHeight = 720;
5516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005517 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005518 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5519 // Frame should not be dropped.
5520 WaitForEncodedFrame(1);
5521
5522 // Trigger QVGA "singlecast"
5523 // Update the config.
5524 VideoEncoderConfig video_encoder_config;
5525 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5526 &video_encoder_config);
5527 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5528 vp9_settings.numberOfSpatialLayers = 3;
5529 // Since only one layer is active - automatic resize should be enabled.
5530 vp9_settings.automaticResizeOn = true;
5531 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005532 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005533 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005534 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005535 video_encoder_config.content_type =
5536 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005537 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005538 // which SVC layers are active.
5539 video_encoder_config.simulcast_layers.resize(3);
5540
5541 video_encoder_config.simulcast_layers[0].active = true;
5542 video_encoder_config.simulcast_layers[1].active = false;
5543 video_encoder_config.simulcast_layers[2].active = false;
5544
5545 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5546 kMaxPayloadLength);
5547 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5548
5549 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5550 // Frame should not be dropped.
5551 WaitForEncodedFrame(2);
5552
5553 // Trigger HD "singlecast"
5554 video_encoder_config.simulcast_layers[0].active = false;
5555 video_encoder_config.simulcast_layers[1].active = false;
5556 video_encoder_config.simulcast_layers[2].active = true;
5557
5558 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5559 kMaxPayloadLength);
5560 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5561
5562 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5563 // Frame should be dropped because of initial frame drop.
5564 ExpectDroppedFrame();
5565
5566 // Expect the sink_wants to specify a scaled frame.
5567 EXPECT_TRUE_WAIT(
5568 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5569 video_stream_encoder_->Stop();
5570}
5571
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005572TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005573 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5574 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5575 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5576 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5577 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5578 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5579 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5580 fake_encoder_.SetResolutionBitrateLimits(
5581 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5582
5583 VideoEncoderConfig video_encoder_config;
5584 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5585 &video_encoder_config);
5586 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5587 vp9_settings.numberOfSpatialLayers = 3;
5588 // Since only one layer is active - automatic resize should be enabled.
5589 vp9_settings.automaticResizeOn = true;
5590 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005591 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005592 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005593 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005594 video_encoder_config.content_type =
5595 VideoEncoderConfig::ContentType::kRealtimeVideo;
5596 // Simulcast layers are used to indicate which spatial layers are active.
5597 video_encoder_config.simulcast_layers.resize(3);
5598 video_encoder_config.simulcast_layers[0].active = false;
5599 video_encoder_config.simulcast_layers[1].active = true;
5600 video_encoder_config.simulcast_layers[2].active = false;
5601
5602 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5603 kMaxPayloadLength);
5604 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5605
5606 // The encoder bitrate limits for 360p should be used.
5607 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5608 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005609 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5610 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5611 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5612 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5613 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5614 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005615 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005616 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005617 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005618 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005619
5620 // The encoder bitrate limits for 270p should be used.
5621 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5622 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005623 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5624 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5625 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5626 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5627 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5628 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005629 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005630 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005631 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005632 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005633
5634 video_stream_encoder_->Stop();
5635}
5636
5637TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005638 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5639 VideoEncoderConfig video_encoder_config;
5640 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5641 &video_encoder_config);
5642 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5643 vp9_settings.numberOfSpatialLayers = 3;
5644 // Since only one layer is active - automatic resize should be enabled.
5645 vp9_settings.automaticResizeOn = true;
5646 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005647 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005648 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005649 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005650 video_encoder_config.content_type =
5651 VideoEncoderConfig::ContentType::kRealtimeVideo;
5652 // Simulcast layers are used to indicate which spatial layers are active.
5653 video_encoder_config.simulcast_layers.resize(3);
5654 video_encoder_config.simulcast_layers[0].active = false;
5655 video_encoder_config.simulcast_layers[1].active = true;
5656 video_encoder_config.simulcast_layers[2].active = false;
5657
5658 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5659 kMaxPayloadLength);
5660 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5661
5662 // The default bitrate limits for 360p should be used.
5663 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005664 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5665 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005666 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5667 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005668 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5669 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5670 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5671 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5672 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5673 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005674 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005675 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005676 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005677 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005678
5679 // The default bitrate limits for 270p should be used.
5680 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005681 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5682 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005683 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5684 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005685 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5686 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5687 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5688 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5689 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5690 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005691 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005692 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005693 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005694 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005695
5696 video_stream_encoder_->Stop();
5697}
5698
5699TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5700 webrtc::test::ScopedFieldTrials field_trials(
5701 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5702 VideoEncoderConfig video_encoder_config;
5703 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5704 &video_encoder_config);
5705 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5706 vp9_settings.numberOfSpatialLayers = 3;
5707 // Since only one layer is active - automatic resize should be enabled.
5708 vp9_settings.automaticResizeOn = true;
5709 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005710 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005711 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005712 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005713 video_encoder_config.content_type =
5714 VideoEncoderConfig::ContentType::kRealtimeVideo;
5715 // Simulcast layers are used to indicate which spatial layers are active.
5716 video_encoder_config.simulcast_layers.resize(3);
5717 video_encoder_config.simulcast_layers[0].active = false;
5718 video_encoder_config.simulcast_layers[1].active = true;
5719 video_encoder_config.simulcast_layers[2].active = false;
5720
5721 // Reset encoder for field trials to take effect.
5722 ConfigureEncoder(video_encoder_config.Copy());
5723
5724 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5725 kMaxPayloadLength);
5726 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5727
5728 // The default bitrate limits for 360p should not be used.
5729 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005730 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5731 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005732 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5733 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005734 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5735 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5736 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5737 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5738 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5739 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005740 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005741 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005742
5743 video_stream_encoder_->Stop();
5744}
5745
5746TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5747 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5748 /*num_spatial_layers=*/1, /*screenshare=*/false);
5749
5750 // The default singlecast bitrate limits for 720p should not be used.
5751 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005752 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5753 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005754 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5755 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005756 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5757 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5758 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5759 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5760 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5761 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005762 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005763 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005764
5765 video_stream_encoder_->Stop();
5766}
5767
5768TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005769 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5770 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5771 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5772 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5773 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5774 fake_encoder_.SetResolutionBitrateLimits(
5775 {kEncoderLimits180p, kEncoderLimits720p});
5776
5777 VideoEncoderConfig video_encoder_config;
5778 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5779 &video_encoder_config);
5780 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5781 vp9_settings.numberOfSpatialLayers = 3;
5782 // Since only one layer is active - automatic resize should be enabled.
5783 vp9_settings.automaticResizeOn = true;
5784 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005785 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005786 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005787 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005788 video_encoder_config.content_type =
5789 VideoEncoderConfig::ContentType::kRealtimeVideo;
5790 // Simulcast layers are used to indicate which spatial layers are active.
5791 video_encoder_config.simulcast_layers.resize(3);
5792 video_encoder_config.simulcast_layers[0].active = true;
5793 video_encoder_config.simulcast_layers[1].active = false;
5794 video_encoder_config.simulcast_layers[2].active = false;
5795
5796 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5797 kMaxPayloadLength);
5798 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5799
5800 // Limits not applied on lowest stream, limits for 180p should not be used.
5801 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5802 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5804 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5805 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5806 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5807 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5808 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005809 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005810 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005811 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005812 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005813
5814 video_stream_encoder_->Stop();
5815}
5816
5817TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005818 InitialFrameDropActivatesWhenResolutionIncreases) {
5819 const int kWidth = 640;
5820 const int kHeight = 360;
5821
5822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005823 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005824 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5825 // Frame should not be dropped.
5826 WaitForEncodedFrame(1);
5827
5828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005829 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005830 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5831 // Frame should not be dropped, bitrate not too low for frame.
5832 WaitForEncodedFrame(2);
5833
5834 // Incoming resolution increases.
5835 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5836 // Expect to drop this frame, bitrate too low for frame.
5837 ExpectDroppedFrame();
5838
5839 // Expect the sink_wants to specify a scaled frame.
5840 EXPECT_TRUE_WAIT(
5841 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5842 video_stream_encoder_->Stop();
5843}
5844
5845TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5846 const int kWidth = 640;
5847 const int kHeight = 360;
5848 // So that quality scaling doesn't happen by itself.
5849 fake_encoder_.SetQp(kQpHigh);
5850
5851 AdaptingFrameForwarder source(&time_controller_);
5852 source.set_adaptation_enabled(true);
5853 video_stream_encoder_->SetSource(
5854 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5855
5856 int timestamp = 1;
5857
5858 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005859 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005860 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5861 WaitForEncodedFrame(timestamp);
5862 timestamp += 9000;
5863 // Long pause to disable all first BWE drop logic.
5864 AdvanceTime(TimeDelta::Millis(1000));
5865
5866 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005867 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005868 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5869 // Not dropped frame, as initial frame drop is disabled by now.
5870 WaitForEncodedFrame(timestamp);
5871 timestamp += 9000;
5872 AdvanceTime(TimeDelta::Millis(100));
5873
5874 // Quality adaptation down.
5875 video_stream_encoder_->TriggerQualityLow();
5876
5877 // Adaptation has an effect.
5878 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5879 5000);
5880
5881 // Frame isn't dropped as initial frame dropper is disabled.
5882 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5883 WaitForEncodedFrame(timestamp);
5884 timestamp += 9000;
5885 AdvanceTime(TimeDelta::Millis(100));
5886
5887 // Quality adaptation up.
5888 video_stream_encoder_->TriggerQualityHigh();
5889
5890 // Adaptation has an effect.
5891 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5892 5000);
5893
5894 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5895 // Frame should not be dropped, as initial framedropper is off.
5896 WaitForEncodedFrame(timestamp);
5897
5898 video_stream_encoder_->Stop();
5899}
5900
Åsa Persson7f354f82021-02-04 15:52:15 +01005901TEST_F(VideoStreamEncoderTest,
5902 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5903 const int kMinStartBps360p = 222000;
5904 fake_encoder_.SetResolutionBitrateLimits(
5905 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5906 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5907 800000)});
5908
5909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5910 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5911 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5912 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5913 0, 0, 0);
5914 // Frame should not be dropped, bitrate not too low for frame.
5915 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5916 WaitForEncodedFrame(1);
5917
5918 // Incoming resolution increases, initial frame drop activates.
5919 // Frame should be dropped, link allocation too low for frame.
5920 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5921 ExpectDroppedFrame();
5922
5923 // Expect sink_wants to specify a scaled frame.
5924 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5925 5000);
5926 video_stream_encoder_->Stop();
5927}
5928
5929TEST_F(VideoStreamEncoderTest,
5930 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5931 const int kMinStartBps360p = 222000;
5932 fake_encoder_.SetResolutionBitrateLimits(
5933 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5934 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5935 800000)});
5936
5937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5938 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5939 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5940 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5941 0, 0, 0);
5942 // Frame should not be dropped, bitrate not too low for frame.
5943 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5944 WaitForEncodedFrame(1);
5945
5946 // Incoming resolution increases, initial frame drop activates.
5947 // Frame should be dropped, link allocation not too low for frame.
5948 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5949 WaitForEncodedFrame(2);
5950
5951 video_stream_encoder_->Stop();
5952}
5953
Åsa Perssone644a032019-11-08 15:56:00 +01005954TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5955 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02005956 "WebRTC-Video-QualityRampupSettings/"
5957 "min_pixels:921600,min_duration_ms:2000/");
5958
5959 const int kWidth = 1280;
5960 const int kHeight = 720;
5961 const int kFps = 10;
5962 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01005963
5964 // Reset encoder for field trials to take effect.
5965 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02005966 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02005967 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005968 ConfigureEncoder(std::move(config));
5969 fake_encoder_.SetQp(kQpLow);
5970
5971 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005972 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005973 source.set_adaptation_enabled(true);
5974 video_stream_encoder_->SetSource(&source,
5975 DegradationPreference::MAINTAIN_FRAMERATE);
5976
5977 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02005978 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02005979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005980 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005981
5982 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02005983 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01005984 int64_t timestamp_ms = kFrameIntervalMs;
5985 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5986 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005987 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5988 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005989
5990 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5992 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005993
Artem Titovab30d722021-07-27 16:22:11 +02005994 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02005995 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01005996 for (size_t i = 1; i <= 10; i++) {
5997 timestamp_ms += kFrameIntervalMs;
5998 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5999 WaitForEncodedFrame(timestamp_ms);
6000 }
Åsa Persson06defc42021-09-10 15:28:48 +02006001
6002 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6003 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6004 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6005 timestamp_ms += kFrameIntervalMs;
6006 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6007 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006008 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6009 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6010
Åsa Persson06defc42021-09-10 15:28:48 +02006011 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006012 timestamp_ms += kFrameIntervalMs;
6013 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6014 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006015 // The ramp-up code involves the adaptation queue, give it time to execute.
6016 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006018 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006019
6020 // Frame should not be adapted.
6021 timestamp_ms += kFrameIntervalMs;
6022 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6023 WaitForEncodedFrame(kWidth, kHeight);
6024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6025
6026 video_stream_encoder_->Stop();
6027}
6028
mflodmancc3d4422017-08-03 08:27:51 -07006029TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006030 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006031 webrtc::test::ScopedFieldTrials field_trials(
6032 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006033 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006034 source.set_adaptation_enabled(true);
6035 video_stream_encoder_->SetSource(&source,
6036 DegradationPreference::MAINTAIN_FRAMERATE);
6037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006038 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006039 fake_encoder_.SetQp(kQpHigh + 1);
6040 const int kWidth = 1280;
6041 const int kHeight = 720;
6042 const int64_t kFrameIntervalMs = 100;
6043 int64_t timestamp_ms = kFrameIntervalMs;
6044 for (size_t i = 1; i <= 100; i++) {
6045 timestamp_ms += kFrameIntervalMs;
6046 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6047 WaitForEncodedFrame(timestamp_ms);
6048 }
6049 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6050 // for the first time.
6051 // TODO(eshr): We should avoid these waits by using threads with simulated
6052 // time.
6053 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6054 2000 * 2.5 * 2);
6055 timestamp_ms += kFrameIntervalMs;
6056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6057 WaitForEncodedFrame(timestamp_ms);
6058 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6059 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6060 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6061
6062 // Disable Quality scaling by turning off scaler on the encoder and
6063 // reconfiguring.
6064 fake_encoder_.SetQualityScaling(false);
6065 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6066 kMaxPayloadLength);
6067 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006068 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006069 // Since we turned off the quality scaler, the adaptations made by it are
6070 // removed.
6071 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6072 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6073
6074 video_stream_encoder_->Stop();
6075}
6076
6077TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006078 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6079 const int kTooSmallWidth = 10;
6080 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006081 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006082 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006083
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006084 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006085 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006086 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006087 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006088 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006089 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6090
6091 // Trigger adapt down, too small frame, expect no change.
6092 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006093 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006094 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006095 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006096 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6097 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6098
mflodmancc3d4422017-08-03 08:27:51 -07006099 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006100}
6101
mflodmancc3d4422017-08-03 08:27:51 -07006102TEST_F(VideoStreamEncoderTest,
6103 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006104 const int kTooSmallWidth = 10;
6105 const int kTooSmallHeight = 10;
6106 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006108 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006109
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006110 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006111 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006112 video_stream_encoder_->SetSource(&source,
6113 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006114 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006115 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6116 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6117
6118 // Trigger adapt down, expect limited framerate.
6119 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006120 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006121 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006122 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006123 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6124 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6125 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6126
6127 // Trigger adapt down, too small frame, expect no change.
6128 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006129 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006130 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006131 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006132 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6133 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6134 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6135
mflodmancc3d4422017-08-03 08:27:51 -07006136 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006137}
6138
mflodmancc3d4422017-08-03 08:27:51 -07006139TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006140 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006141 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006142 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006143 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006144 const int kFrameWidth = 1280;
6145 const int kFrameHeight = 720;
6146 video_source_.IncomingCapturedFrame(
6147 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006148 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006149 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006150}
6151
sprangb1ca0732017-02-01 08:38:12 -08006152// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006153TEST_F(VideoStreamEncoderTest,
6154 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006155 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006156 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006157
6158 const int kFrameWidth = 1280;
6159 const int kFrameHeight = 720;
6160 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006161 // requested by
6162 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006163 video_source_.set_adaptation_enabled(true);
6164
6165 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006166 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006167 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006168
6169 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006170 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006171 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006172 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006173 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006174
asaperssonfab67072017-04-04 05:51:49 -07006175 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006176 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006177 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006178 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006179 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006180
mflodmancc3d4422017-08-03 08:27:51 -07006181 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006182}
sprangfe627f32017-03-29 08:24:59 -07006183
mflodmancc3d4422017-08-03 08:27:51 -07006184TEST_F(VideoStreamEncoderTest,
6185 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006186 const int kFrameWidth = 1280;
6187 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006188
Henrik Boström381d1092020-05-12 18:49:07 +02006189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006190 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006191 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006192 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006193 video_source_.set_adaptation_enabled(true);
6194
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006195 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006196
6197 video_source_.IncomingCapturedFrame(
6198 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006199 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006200
6201 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006202 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006203
6204 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006205 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006206 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006207 video_source_.IncomingCapturedFrame(
6208 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006209 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006210 }
6211
6212 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006213 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006214 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006215 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006216 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006217 video_source_.IncomingCapturedFrame(
6218 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006219 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006220 ++num_frames_dropped;
6221 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006222 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006223 }
6224 }
6225
sprang4847ae62017-06-27 07:06:52 -07006226 // Add some slack to account for frames dropped by the frame dropper.
6227 const int kErrorMargin = 1;
6228 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006229 kErrorMargin);
6230
6231 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006232 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006233 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006234 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006235 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006236 video_source_.IncomingCapturedFrame(
6237 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006238 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006239 ++num_frames_dropped;
6240 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006241 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006242 }
6243 }
sprang4847ae62017-06-27 07:06:52 -07006244 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006245 kErrorMargin);
6246
6247 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006248 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006249 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006250 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006251 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006252 video_source_.IncomingCapturedFrame(
6253 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006254 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006255 ++num_frames_dropped;
6256 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006257 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006258 }
6259 }
sprang4847ae62017-06-27 07:06:52 -07006260 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006261 kErrorMargin);
6262
6263 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006264 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006265 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006266 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006267 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006268 video_source_.IncomingCapturedFrame(
6269 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006270 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006271 ++num_frames_dropped;
6272 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006273 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006274 }
6275 }
6276 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6277
mflodmancc3d4422017-08-03 08:27:51 -07006278 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006279}
6280
mflodmancc3d4422017-08-03 08:27:51 -07006281TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006282 const int kFramerateFps = 5;
6283 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006284 const int kFrameWidth = 1280;
6285 const int kFrameHeight = 720;
6286
sprang4847ae62017-06-27 07:06:52 -07006287 // Reconfigure encoder with two temporal layers and screensharing, which will
6288 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006289 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006290
Henrik Boström381d1092020-05-12 18:49:07 +02006291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006292 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006293 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006294 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006295 video_source_.set_adaptation_enabled(true);
6296
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006297 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006298
6299 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006300 rtc::VideoSinkWants last_wants;
6301 do {
6302 last_wants = video_source_.sink_wants();
6303
sprangc5d62e22017-04-02 23:53:04 -07006304 // Insert frames to get a new fps estimate...
6305 for (int j = 0; j < kFramerateFps; ++j) {
6306 video_source_.IncomingCapturedFrame(
6307 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006308 if (video_source_.last_sent_width()) {
6309 sink_.WaitForEncodedFrame(timestamp_ms);
6310 }
sprangc5d62e22017-04-02 23:53:04 -07006311 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006312 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006313 }
6314 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006315 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006316 } while (video_source_.sink_wants().max_framerate_fps <
6317 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006318
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006319 EXPECT_THAT(video_source_.sink_wants(),
6320 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006321
mflodmancc3d4422017-08-03 08:27:51 -07006322 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006323}
asaperssonf7e294d2017-06-13 23:25:22 -07006324
mflodmancc3d4422017-08-03 08:27:51 -07006325TEST_F(VideoStreamEncoderTest,
6326 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006327 const int kWidth = 1280;
6328 const int kHeight = 720;
6329 const int64_t kFrameIntervalMs = 150;
6330 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006332 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006333
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006334 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006335 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006336 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006337 video_stream_encoder_->SetSource(&source,
6338 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006339 timestamp_ms += kFrameIntervalMs;
6340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006341 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006342 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6344 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6345 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6346
6347 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006348 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006349 timestamp_ms += kFrameIntervalMs;
6350 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006351 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006352 EXPECT_THAT(source.sink_wants(),
6353 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006354 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6356 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6357
6358 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006359 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006360 timestamp_ms += kFrameIntervalMs;
6361 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006362 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006363 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006364 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6365 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6366 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6367
6368 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006369 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006370 timestamp_ms += kFrameIntervalMs;
6371 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006372 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006373 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006374 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6375 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6376 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6377
6378 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006379 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006380 timestamp_ms += kFrameIntervalMs;
6381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006382 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006383 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006384 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6386 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6387
6388 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006389 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006390 timestamp_ms += kFrameIntervalMs;
6391 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006392 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006393 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6395 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6396 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6397
6398 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006399 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006400 timestamp_ms += kFrameIntervalMs;
6401 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006402 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006403 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006404 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6405 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6406 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6407
6408 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006409 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006410 timestamp_ms += kFrameIntervalMs;
6411 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006412 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006413 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006414 rtc::VideoSinkWants last_wants = source.sink_wants();
6415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6416 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6417 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6418
6419 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006420 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006421 timestamp_ms += kFrameIntervalMs;
6422 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006423 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006424 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6426 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6427 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6428
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006429 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006430 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006431 timestamp_ms += kFrameIntervalMs;
6432 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006433 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006434 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006435 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6436 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6437 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6438
6439 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006440 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006441 timestamp_ms += kFrameIntervalMs;
6442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006443 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006444 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006445 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6446 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6447 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6448
6449 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006450 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006451 timestamp_ms += kFrameIntervalMs;
6452 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006453 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006454 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006455 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6456 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6457 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6458
6459 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006460 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006461 timestamp_ms += kFrameIntervalMs;
6462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006463 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006464 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006465 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6467 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6468
6469 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006470 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006471 timestamp_ms += kFrameIntervalMs;
6472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006473 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006474 EXPECT_THAT(source.sink_wants(), FpsMax());
6475 EXPECT_EQ(source.sink_wants().max_pixel_count,
6476 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006477 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6479 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6480
6481 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006482 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006483 timestamp_ms += kFrameIntervalMs;
6484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006486 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6489 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6490
Åsa Persson30ab0152019-08-27 12:22:33 +02006491 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006492 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006493 timestamp_ms += kFrameIntervalMs;
6494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006495 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006496 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006498 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6500 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6501
6502 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006503 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006504 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006505 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6506
mflodmancc3d4422017-08-03 08:27:51 -07006507 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006508}
6509
mflodmancc3d4422017-08-03 08:27:51 -07006510TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006511 const int kWidth = 1280;
6512 const int kHeight = 720;
6513 const int64_t kFrameIntervalMs = 150;
6514 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006516 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006517
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006518 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006519 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006520 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006521 video_stream_encoder_->SetSource(&source,
6522 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006523 timestamp_ms += kFrameIntervalMs;
6524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006525 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006526 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6528 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6529 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6531 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6532 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6533
6534 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006535 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006536 timestamp_ms += kFrameIntervalMs;
6537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006538 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006539 EXPECT_THAT(source.sink_wants(),
6540 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6542 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6543 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6544 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6545 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6546 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6547
6548 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006549 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006550 timestamp_ms += kFrameIntervalMs;
6551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6556 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6557 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6558 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6559 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6560
6561 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006562 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006563 timestamp_ms += kFrameIntervalMs;
6564 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006565 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006566 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6569 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6570 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6571 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6572 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6573
Evan Shrubsole64469032020-06-11 10:45:29 +02006574 // Trigger cpu adapt up, expect no change since QP is most limited.
6575 {
6576 // Store current sink wants since we expect no change and if there is no
6577 // change then last_wants() is not updated.
6578 auto previous_sink_wants = source.sink_wants();
6579 video_stream_encoder_->TriggerCpuUnderuse();
6580 timestamp_ms += kFrameIntervalMs;
6581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6582 WaitForEncodedFrame(timestamp_ms);
6583 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6584 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6585 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6586 }
6587
6588 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6589 video_stream_encoder_->TriggerQualityHigh();
6590 timestamp_ms += kFrameIntervalMs;
6591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6592 WaitForEncodedFrame(timestamp_ms);
6593 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6596 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6597 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6598 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6599 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6600
6601 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6602 // expect increased resolution (960x540@30fps).
6603 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006604 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006605 timestamp_ms += kFrameIntervalMs;
6606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006607 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006608 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6613 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006615
Evan Shrubsole64469032020-06-11 10:45:29 +02006616 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6617 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006618 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006619 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006620 timestamp_ms += kFrameIntervalMs;
6621 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006622 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006623 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006624 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006625 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6626 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6629 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006630 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006631
6632 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006633 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006634 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006635 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006636 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006637
mflodmancc3d4422017-08-03 08:27:51 -07006638 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006639}
6640
mflodmancc3d4422017-08-03 08:27:51 -07006641TEST_F(VideoStreamEncoderTest,
6642 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006643 const int kWidth = 640;
6644 const int kHeight = 360;
6645 const int kFpsLimit = 15;
6646 const int64_t kFrameIntervalMs = 150;
6647 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006650
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006651 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006652 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006653 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006654 video_stream_encoder_->SetSource(&source,
6655 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006656 timestamp_ms += kFrameIntervalMs;
6657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006658 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006659 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6663 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6664 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6665 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6666
6667 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006668 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006669 timestamp_ms += kFrameIntervalMs;
6670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006671 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006672 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6676 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6677 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6679
6680 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006681 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006682 timestamp_ms += kFrameIntervalMs;
6683 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006684 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006685 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006686 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006687 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6689 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6690 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6691 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6692
Evan Shrubsole64469032020-06-11 10:45:29 +02006693 // Trigger cpu adapt up, expect no change because quality is most limited.
6694 {
6695 auto previous_sink_wants = source.sink_wants();
6696 // Store current sink wants since we expect no change ind if there is no
6697 // change then last__wants() is not updated.
6698 video_stream_encoder_->TriggerCpuUnderuse();
6699 timestamp_ms += kFrameIntervalMs;
6700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6701 WaitForEncodedFrame(timestamp_ms);
6702 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6703 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6704 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6705 }
6706
6707 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6708 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006709 timestamp_ms += kFrameIntervalMs;
6710 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006711 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006712 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6714 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6715 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006716 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6717 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6718 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006719
Evan Shrubsole64469032020-06-11 10:45:29 +02006720 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006721 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006722 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006723 timestamp_ms += kFrameIntervalMs;
6724 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006725 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006726 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006727 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6728 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6730 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6731 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006732 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006733
6734 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006735 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006736 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006737 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006738 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006739
mflodmancc3d4422017-08-03 08:27:51 -07006740 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006741}
6742
mflodmancc3d4422017-08-03 08:27:51 -07006743TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006744 const int kFrameWidth = 1920;
6745 const int kFrameHeight = 1080;
6746 // 3/4 of 1920.
6747 const int kAdaptedFrameWidth = 1440;
6748 // 3/4 of 1080 rounded down to multiple of 4.
6749 const int kAdaptedFrameHeight = 808;
6750 const int kFramerate = 24;
6751
Henrik Boström381d1092020-05-12 18:49:07 +02006752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006753 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006754 // Trigger reconfigure encoder (without resetting the entire instance).
6755 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006756 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6757 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006758 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006759 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006760 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006761 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006762 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006763 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006764
6765 video_source_.set_adaptation_enabled(true);
6766
6767 video_source_.IncomingCapturedFrame(
6768 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006769 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006770
6771 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006772 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006773 video_source_.IncomingCapturedFrame(
6774 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006775 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006776
mflodmancc3d4422017-08-03 08:27:51 -07006777 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006778}
6779
mflodmancc3d4422017-08-03 08:27:51 -07006780TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006781 const int kFrameWidth = 1280;
6782 const int kFrameHeight = 720;
6783 const int kLowFps = 2;
6784 const int kHighFps = 30;
6785
Henrik Boström381d1092020-05-12 18:49:07 +02006786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006787 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006788
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006789 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006790 max_framerate_ = kLowFps;
6791
6792 // Insert 2 seconds of 2fps video.
6793 for (int i = 0; i < kLowFps * 2; ++i) {
6794 video_source_.IncomingCapturedFrame(
6795 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6796 WaitForEncodedFrame(timestamp_ms);
6797 timestamp_ms += 1000 / kLowFps;
6798 }
6799
6800 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006802 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006803 video_source_.IncomingCapturedFrame(
6804 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6805 WaitForEncodedFrame(timestamp_ms);
6806 timestamp_ms += 1000 / kLowFps;
6807
6808 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6809
6810 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006811 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006812 const int kFrameIntervalMs = 1000 / kHighFps;
6813 max_framerate_ = kHighFps;
6814 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6815 video_source_.IncomingCapturedFrame(
6816 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6817 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6818 // be dropped if the encoder hans't been updated with the new higher target
6819 // framerate yet, causing it to overshoot the target bitrate and then
6820 // suffering the wrath of the media optimizer.
6821 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6822 timestamp_ms += kFrameIntervalMs;
6823 }
6824
6825 // Don expect correct measurement just yet, but it should be higher than
6826 // before.
6827 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6828
mflodmancc3d4422017-08-03 08:27:51 -07006829 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006830}
6831
mflodmancc3d4422017-08-03 08:27:51 -07006832TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006833 const int kFrameWidth = 1280;
6834 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006835 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006836 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006837 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006838
Henrik Boström381d1092020-05-12 18:49:07 +02006839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006840 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006841 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006842
6843 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006844 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006845 video_source_.IncomingCapturedFrame(
6846 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6847 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006848 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006849
6850 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006852 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006853
6854 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006855 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006856 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006857
Per Kjellanderdcef6412020-10-07 15:09:05 +02006858 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006859 video_source_.IncomingCapturedFrame(
6860 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6861 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006862 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006863
mflodmancc3d4422017-08-03 08:27:51 -07006864 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006865}
ilnik6b826ef2017-06-16 06:53:48 -07006866
Niels Möller4db138e2018-04-19 09:04:13 +02006867TEST_F(VideoStreamEncoderTest,
6868 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6869 const int kFrameWidth = 1280;
6870 const int kFrameHeight = 720;
6871 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006873 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006874 video_source_.IncomingCapturedFrame(
6875 CreateFrame(1, kFrameWidth, kFrameHeight));
6876 WaitForEncodedFrame(1);
6877 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6878 .low_encode_usage_threshold_percent,
6879 default_options.low_encode_usage_threshold_percent);
6880 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6881 .high_encode_usage_threshold_percent,
6882 default_options.high_encode_usage_threshold_percent);
6883 video_stream_encoder_->Stop();
6884}
6885
6886TEST_F(VideoStreamEncoderTest,
6887 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6888 const int kFrameWidth = 1280;
6889 const int kFrameHeight = 720;
6890 CpuOveruseOptions hardware_options;
6891 hardware_options.low_encode_usage_threshold_percent = 150;
6892 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006893 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006894
Henrik Boström381d1092020-05-12 18:49:07 +02006895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006897 video_source_.IncomingCapturedFrame(
6898 CreateFrame(1, kFrameWidth, kFrameHeight));
6899 WaitForEncodedFrame(1);
6900 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6901 .low_encode_usage_threshold_percent,
6902 hardware_options.low_encode_usage_threshold_percent);
6903 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6904 .high_encode_usage_threshold_percent,
6905 hardware_options.high_encode_usage_threshold_percent);
6906 video_stream_encoder_->Stop();
6907}
6908
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006909TEST_F(VideoStreamEncoderTest,
6910 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6911 const int kFrameWidth = 1280;
6912 const int kFrameHeight = 720;
6913
6914 const CpuOveruseOptions default_options;
6915 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006916 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006917 video_source_.IncomingCapturedFrame(
6918 CreateFrame(1, kFrameWidth, kFrameHeight));
6919 WaitForEncodedFrame(1);
6920 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6921 .low_encode_usage_threshold_percent,
6922 default_options.low_encode_usage_threshold_percent);
6923 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6924 .high_encode_usage_threshold_percent,
6925 default_options.high_encode_usage_threshold_percent);
6926
6927 CpuOveruseOptions hardware_options;
6928 hardware_options.low_encode_usage_threshold_percent = 150;
6929 hardware_options.high_encode_usage_threshold_percent = 200;
6930 fake_encoder_.SetIsHardwareAccelerated(true);
6931
6932 video_source_.IncomingCapturedFrame(
6933 CreateFrame(2, kFrameWidth, kFrameHeight));
6934 WaitForEncodedFrame(2);
6935
6936 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6937 .low_encode_usage_threshold_percent,
6938 hardware_options.low_encode_usage_threshold_percent);
6939 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6940 .high_encode_usage_threshold_percent,
6941 hardware_options.high_encode_usage_threshold_percent);
6942
6943 video_stream_encoder_->Stop();
6944}
6945
Niels Möller6bb5ab92019-01-11 11:11:10 +01006946TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6947 const int kFrameWidth = 320;
6948 const int kFrameHeight = 240;
6949 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02006950 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006951 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6952
Henrik Boström381d1092020-05-12 18:49:07 +02006953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006954 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006955
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006956 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006957 max_framerate_ = kFps;
6958
6959 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6960 fake_encoder_.SimulateOvershoot(1.0);
6961 int num_dropped = 0;
6962 for (int i = 0; i < kNumFramesInRun; ++i) {
6963 video_source_.IncomingCapturedFrame(
6964 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6965 // Wait up to two frame durations for a frame to arrive.
6966 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6967 ++num_dropped;
6968 }
6969 timestamp_ms += 1000 / kFps;
6970 }
6971
Erik Språnga8d48ab2019-02-08 14:17:40 +01006972 // Framerate should be measured to be near the expected target rate.
6973 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6974
6975 // Frame drops should be within 5% of expected 0%.
6976 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006977
6978 // Make encoder produce frames at double the expected bitrate during 3 seconds
6979 // of video, verify number of drops. Rate needs to be slightly changed in
6980 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006981 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006982 const RateControlSettings trials =
6983 RateControlSettings::ParseFromFieldTrials();
6984 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006985 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006986 // frame dropping since the adjuter will try to just lower the target
6987 // bitrate rather than drop frames. If network headroom can be used, it
6988 // doesn't push back as hard so we don't need quite as much overshoot.
6989 // These numbers are unfortunately a bit magical but there's not trivial
6990 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01006991 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01006992 }
6993 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006995 kTargetBitrate + DataRate::KilobitsPerSec(1),
6996 kTargetBitrate + DataRate::KilobitsPerSec(1),
6997 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006998 num_dropped = 0;
6999 for (int i = 0; i < kNumFramesInRun; ++i) {
7000 video_source_.IncomingCapturedFrame(
7001 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7002 // Wait up to two frame durations for a frame to arrive.
7003 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7004 ++num_dropped;
7005 }
7006 timestamp_ms += 1000 / kFps;
7007 }
7008
Henrik Boström381d1092020-05-12 18:49:07 +02007009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007010 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007011
7012 // Target framerate should be still be near the expected target, despite
7013 // the frame drops.
7014 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7015
7016 // Frame drops should be within 5% of expected 50%.
7017 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007018
7019 video_stream_encoder_->Stop();
7020}
7021
7022TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7023 const int kFrameWidth = 320;
7024 const int kFrameHeight = 240;
7025 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007026 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007027
7028 ASSERT_GT(max_framerate_, kActualInputFps);
7029
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007030 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007031 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007034
7035 // Insert 3 seconds of video, with an input fps lower than configured max.
7036 for (int i = 0; i < kActualInputFps * 3; ++i) {
7037 video_source_.IncomingCapturedFrame(
7038 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7039 // Wait up to two frame durations for a frame to arrive.
7040 WaitForEncodedFrame(timestamp_ms);
7041 timestamp_ms += 1000 / kActualInputFps;
7042 }
7043
7044 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7045
7046 video_stream_encoder_->Stop();
7047}
7048
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007049TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007050 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007053
7054 fake_encoder_.BlockNextEncode();
7055 video_source_.IncomingCapturedFrame(
7056 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7057 WaitForEncodedFrame(1);
7058 // On the very first frame full update should be forced.
7059 rect = fake_encoder_.GetLastUpdateRect();
7060 EXPECT_EQ(rect.offset_x, 0);
7061 EXPECT_EQ(rect.offset_y, 0);
7062 EXPECT_EQ(rect.height, codec_height_);
7063 EXPECT_EQ(rect.width, codec_width_);
7064 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7065 // call to ContinueEncode.
7066 video_source_.IncomingCapturedFrame(
7067 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7068 ExpectDroppedFrame();
7069 video_source_.IncomingCapturedFrame(
7070 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7071 ExpectDroppedFrame();
7072 fake_encoder_.ContinueEncode();
7073 WaitForEncodedFrame(3);
7074 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7075 rect = fake_encoder_.GetLastUpdateRect();
7076 EXPECT_EQ(rect.offset_x, 1);
7077 EXPECT_EQ(rect.offset_y, 0);
7078 EXPECT_EQ(rect.width, 10);
7079 EXPECT_EQ(rect.height, 1);
7080
7081 video_source_.IncomingCapturedFrame(
7082 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7083 WaitForEncodedFrame(4);
7084 // Previous frame was encoded, so no accumulation should happen.
7085 rect = fake_encoder_.GetLastUpdateRect();
7086 EXPECT_EQ(rect.offset_x, 0);
7087 EXPECT_EQ(rect.offset_y, 0);
7088 EXPECT_EQ(rect.width, 1);
7089 EXPECT_EQ(rect.height, 1);
7090
7091 video_stream_encoder_->Stop();
7092}
7093
Erik Språngd7329ca2019-02-21 21:19:53 +01007094TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007096 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007097
7098 // First frame is always keyframe.
7099 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7100 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007101 EXPECT_THAT(
7102 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007103 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007104
7105 // Insert delta frame.
7106 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7107 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007108 EXPECT_THAT(
7109 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007110 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007111
7112 // Request next frame be a key-frame.
7113 video_stream_encoder_->SendKeyFrame();
7114 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7115 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007116 EXPECT_THAT(
7117 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007118 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007119
7120 video_stream_encoder_->Stop();
7121}
7122
7123TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7124 // Setup simulcast with three streams.
7125 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007127 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7128 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007129 // Wait for all three layers before triggering event.
7130 sink_.SetNumExpectedLayers(3);
7131
7132 // First frame is always keyframe.
7133 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7134 WaitForEncodedFrame(1);
7135 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007136 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7137 VideoFrameType::kVideoFrameKey,
7138 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007139
7140 // Insert delta frame.
7141 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7142 WaitForEncodedFrame(2);
7143 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007144 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7145 VideoFrameType::kVideoFrameDelta,
7146 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007147
7148 // Request next frame be a key-frame.
7149 // Only first stream is configured to produce key-frame.
7150 video_stream_encoder_->SendKeyFrame();
7151 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7152 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007153
7154 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7155 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007156 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007157 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007158 VideoFrameType::kVideoFrameKey,
7159 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007160
7161 video_stream_encoder_->Stop();
7162}
7163
7164TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7165 // Configure internal source factory and setup test again.
7166 encoder_factory_.SetHasInternalSource(true);
7167 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007168 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007169 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007170
7171 // Call encoder directly, simulating internal source where encoded frame
7172 // callback in VideoStreamEncoder is called despite no OnFrame().
7173 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7174 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007175 EXPECT_THAT(
7176 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007177 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007178
Niels Möller8f7ce222019-03-21 15:43:58 +01007179 const std::vector<VideoFrameType> kDeltaFrame = {
7180 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007181 // Need to set timestamp manually since manually for injected frame.
7182 VideoFrame frame = CreateFrame(101, nullptr);
7183 frame.set_timestamp(101);
7184 fake_encoder_.InjectFrame(frame, false);
7185 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007186 EXPECT_THAT(
7187 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007188 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007189
7190 // Request key-frame. The forces a dummy frame down into the encoder.
7191 fake_encoder_.ExpectNullFrame();
7192 video_stream_encoder_->SendKeyFrame();
7193 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007194 EXPECT_THAT(
7195 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007196 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007197
7198 video_stream_encoder_->Stop();
7199}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007200
7201TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7202 // Configure internal source factory and setup test again.
7203 encoder_factory_.SetHasInternalSource(true);
7204 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007206 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007207
7208 int64_t timestamp = 1;
7209 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007210 image.capture_time_ms_ = ++timestamp;
7211 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7212 const int64_t kEncodeFinishDelayMs = 10;
7213 image.timing_.encode_start_ms = timestamp;
7214 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007215 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007216 // Wait for frame without incrementing clock.
7217 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7218 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7219 // capture timestamp should be kEncodeFinishDelayMs in the past.
7220 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007221 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007222
7223 video_stream_encoder_->Stop();
7224}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007225
7226TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007227 // SPS contains VUI with restrictions on the maximum number of reordered
7228 // pictures, there is no need to rewrite the bitstream to enable faster
7229 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007230 ResetEncoder("H264", 1, 1, 1, false);
7231
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007233 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007234 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007235
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007236 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007237 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007238
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007239 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7240 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007241
7242 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007243 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007244
7245 video_stream_encoder_->Stop();
7246}
7247
7248TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007249 // SPS does not contain VUI, the bitstream is will be rewritten with added
7250 // VUI with restrictions on the maximum number of reordered pictures to
7251 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007252 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7253 0x00, 0x00, 0x03, 0x03, 0xF4,
7254 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007255 ResetEncoder("H264", 1, 1, 1, false);
7256
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007258 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007259 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007260
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007261 fake_encoder_.SetEncodedImageData(
7262 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007263
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007264 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7265 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007266
7267 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007268 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007269
7270 video_stream_encoder_->Stop();
7271}
7272
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007273TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7274 const int kFrameWidth = 1280;
7275 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007276 const DataRate kTargetBitrate =
7277 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007278
Henrik Boström381d1092020-05-12 18:49:07 +02007279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007280 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007281 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7282
7283 // Insert a first video frame. It should be dropped because of downscale in
7284 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007285 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007286 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7287 frame.set_rotation(kVideoRotation_270);
7288 video_source_.IncomingCapturedFrame(frame);
7289
7290 ExpectDroppedFrame();
7291
7292 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007293 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007294 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7295 frame.set_rotation(kVideoRotation_90);
7296 video_source_.IncomingCapturedFrame(frame);
7297
7298 WaitForEncodedFrame(timestamp_ms);
7299 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7300
7301 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007302 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007303 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7304 frame.set_rotation(kVideoRotation_180);
7305 video_source_.IncomingCapturedFrame(frame);
7306
7307 WaitForEncodedFrame(timestamp_ms);
7308 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7309
7310 video_stream_encoder_->Stop();
7311}
7312
Erik Språng5056af02019-09-02 15:53:11 +02007313TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7314 const int kFrameWidth = 320;
7315 const int kFrameHeight = 180;
7316
7317 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007319 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7320 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7321 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007322 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007323 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007324 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007325
7326 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007327 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007328 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7329 frame.set_rotation(kVideoRotation_270);
7330 video_source_.IncomingCapturedFrame(frame);
7331 WaitForEncodedFrame(timestamp_ms);
7332
7333 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007334 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007335 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7336 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007337 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007338 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007339 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007340 /*link_allocation=*/target_rate,
7341 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007342 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007343 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007344 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7345
7346 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7347 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7348 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007349 DataRate allocation_sum =
7350 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007351 EXPECT_EQ(min_rate, allocation_sum);
7352 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7353
7354 video_stream_encoder_->Stop();
7355}
7356
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007357TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007359 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007360 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007361 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007362 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7363 WaitForEncodedFrame(1);
7364
7365 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7366 ASSERT_TRUE(prev_rate_settings.has_value());
7367 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7368 kDefaultFramerate);
7369
7370 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7371 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7372 timestamp_ms += 1000 / kDefaultFramerate;
7373 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7374 WaitForEncodedFrame(timestamp_ms);
7375 }
7376 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7377 kDefaultFramerate);
7378 // Capture larger frame to trigger a reconfigure.
7379 codec_height_ *= 2;
7380 codec_width_ *= 2;
7381 timestamp_ms += 1000 / kDefaultFramerate;
7382 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7383 WaitForEncodedFrame(timestamp_ms);
7384
7385 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7386 auto current_rate_settings =
7387 fake_encoder_.GetAndResetLastRateControlSettings();
7388 // Ensure we have actually reconfigured twice
7389 // The rate settings should have been set again even though
7390 // they haven't changed.
7391 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007392 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007393
7394 video_stream_encoder_->Stop();
7395}
7396
philipeld9cc8c02019-09-16 14:53:40 +02007397struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007398 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7399 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7400 MOCK_METHOD(void,
7401 RequestEncoderSwitch,
7402 (const webrtc::SdpVideoFormat& format),
7403 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007404};
7405
philipel9b058032020-02-10 11:30:00 +01007406TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7407 constexpr int kDontCare = 100;
7408 StrictMock<MockEncoderSelector> encoder_selector;
7409 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7410 &fake_encoder_, &encoder_selector);
7411 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7412
7413 // Reset encoder for new configuration to take effect.
7414 ConfigureEncoder(video_encoder_config_.Copy());
7415
7416 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7417
7418 video_source_.IncomingCapturedFrame(
7419 CreateFrame(kDontCare, kDontCare, kDontCare));
7420 video_stream_encoder_->Stop();
7421
7422 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7423 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007424 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7425 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007426 video_stream_encoder_.reset();
7427}
7428
7429TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7430 constexpr int kDontCare = 100;
7431
7432 NiceMock<MockEncoderSelector> encoder_selector;
7433 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7434 video_send_config_.encoder_settings.encoder_switch_request_callback =
7435 &switch_callback;
7436 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7437 &fake_encoder_, &encoder_selector);
7438 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7439
7440 // Reset encoder for new configuration to take effect.
7441 ConfigureEncoder(video_encoder_config_.Copy());
7442
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007443 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007444 .WillByDefault(Return(SdpVideoFormat("AV1")));
7445 EXPECT_CALL(switch_callback,
7446 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7447 Field(&SdpVideoFormat::name, "AV1"))));
7448
Henrik Boström381d1092020-05-12 18:49:07 +02007449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007450 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7451 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7452 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007453 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007454 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007455 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007456 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007457
7458 video_stream_encoder_->Stop();
7459}
7460
7461TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7462 constexpr int kSufficientBitrateToNotDrop = 1000;
7463 constexpr int kDontCare = 100;
7464
7465 NiceMock<MockVideoEncoder> video_encoder;
7466 NiceMock<MockEncoderSelector> encoder_selector;
7467 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7468 video_send_config_.encoder_settings.encoder_switch_request_callback =
7469 &switch_callback;
7470 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7471 &video_encoder, &encoder_selector);
7472 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7473
7474 // Reset encoder for new configuration to take effect.
7475 ConfigureEncoder(video_encoder_config_.Copy());
7476
7477 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7478 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7479 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007481 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7482 /*stable_target_bitrate=*/
7483 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7484 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007485 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007486 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007487 /*cwnd_reduce_ratio=*/0);
7488
7489 ON_CALL(video_encoder, Encode(_, _))
7490 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7491 ON_CALL(encoder_selector, OnEncoderBroken())
7492 .WillByDefault(Return(SdpVideoFormat("AV2")));
7493
7494 rtc::Event encode_attempted;
7495 EXPECT_CALL(switch_callback,
7496 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7497 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7498 EXPECT_EQ(format.name, "AV2");
7499 encode_attempted.Set();
7500 });
7501
7502 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7503 encode_attempted.Wait(3000);
7504
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007505 AdvanceTime(TimeDelta::Millis(0));
7506
philipel9b058032020-02-10 11:30:00 +01007507 video_stream_encoder_->Stop();
7508
7509 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7510 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007511 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7512 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007513 video_stream_encoder_.reset();
7514}
7515
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007516TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007517 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007518 const int kFrameWidth = 320;
7519 const int kFrameHeight = 180;
7520
7521 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007522 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007523 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007524 /*target_bitrate=*/rate,
7525 /*stable_target_bitrate=*/rate,
7526 /*link_allocation=*/rate,
7527 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007528 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007529 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007530
7531 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007532 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007533 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7534 frame.set_rotation(kVideoRotation_270);
7535 video_source_.IncomingCapturedFrame(frame);
7536 WaitForEncodedFrame(timestamp_ms);
7537 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7538
7539 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007540 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007541 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007542 /*target_bitrate=*/new_stable_rate,
7543 /*stable_target_bitrate=*/new_stable_rate,
7544 /*link_allocation=*/rate,
7545 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007546 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007547 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007548 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7549 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7550 video_stream_encoder_->Stop();
7551}
7552
7553TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007554 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007555 const int kFrameWidth = 320;
7556 const int kFrameHeight = 180;
7557
7558 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007559 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007561 /*target_bitrate=*/rate,
7562 /*stable_target_bitrate=*/rate,
7563 /*link_allocation=*/rate,
7564 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007565 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007566 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007567
7568 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007569 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007570 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7571 frame.set_rotation(kVideoRotation_270);
7572 video_source_.IncomingCapturedFrame(frame);
7573 WaitForEncodedFrame(timestamp_ms);
7574 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7575
7576 // Set a higher target rate without changing the link_allocation. Should not
7577 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007578 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007580 /*target_bitrate=*/rate,
7581 /*stable_target_bitrate=*/new_stable_rate,
7582 /*link_allocation=*/rate,
7583 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007584 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007585 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007586 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7587 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7588 video_stream_encoder_->Stop();
7589}
7590
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007591TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7592 test::ScopedFieldTrials field_trials(
7593 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7594 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7595 const int kFramerateFps = 30;
7596 const int kWidth = 1920;
7597 const int kHeight = 1080;
7598 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7599 // Works on screenshare mode.
7600 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7601 // We rely on the automatic resolution adaptation, but we handle framerate
7602 // adaptation manually by mocking the stats proxy.
7603 video_source_.set_adaptation_enabled(true);
7604
7605 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007606 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007607 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007608 video_stream_encoder_->SetSource(&video_source_,
7609 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007610 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007611
7612 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7613 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7614
7615 // Pass enough frames with the full update to trigger animation detection.
7616 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007617 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007618 frame.set_ntp_time_ms(timestamp_ms);
7619 frame.set_timestamp_us(timestamp_ms * 1000);
7620 video_source_.IncomingCapturedFrame(frame);
7621 WaitForEncodedFrame(timestamp_ms);
7622 }
7623
7624 // Resolution should be limited.
7625 rtc::VideoSinkWants expected;
7626 expected.max_framerate_fps = kFramerateFps;
7627 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007628 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007629
7630 // Pass one frame with no known update.
7631 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007632 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007633 frame.set_ntp_time_ms(timestamp_ms);
7634 frame.set_timestamp_us(timestamp_ms * 1000);
7635 frame.clear_update_rect();
7636
7637 video_source_.IncomingCapturedFrame(frame);
7638 WaitForEncodedFrame(timestamp_ms);
7639
7640 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007641 EXPECT_THAT(video_source_.sink_wants(),
7642 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007643
7644 video_stream_encoder_->Stop();
7645}
7646
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007647TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7648 const int kWidth = 720; // 540p adapted down.
7649 const int kHeight = 405;
7650 const int kNumFrames = 3;
7651 // Works on screenshare mode.
7652 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7653 /*num_spatial_layers=*/2, /*screenshare=*/true);
7654
7655 video_source_.set_adaptation_enabled(true);
7656
7657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007658 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007659
7660 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7661
7662 // Pass enough frames with the full update to trigger animation detection.
7663 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007664 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007665 frame.set_ntp_time_ms(timestamp_ms);
7666 frame.set_timestamp_us(timestamp_ms * 1000);
7667 video_source_.IncomingCapturedFrame(frame);
7668 WaitForEncodedFrame(timestamp_ms);
7669 }
7670
7671 video_stream_encoder_->Stop();
7672}
7673
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007674TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7675 const float downscale_factors[] = {4.0, 2.0, 1.0};
7676 const int number_layers =
7677 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7678 VideoEncoderConfig config;
7679 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7680 for (int i = 0; i < number_layers; ++i) {
7681 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7682 config.simulcast_layers[i].active = true;
7683 }
7684 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007685 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007686 "VP8", /*max qp*/ 56, /*screencast*/ false,
7687 /*screenshare enabled*/ false);
7688 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007689 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7690 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007691
7692 // First initialization.
7693 // Encoder should be initialized. Next frame should be key frame.
7694 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7695 sink_.SetNumExpectedLayers(number_layers);
7696 int64_t timestamp_ms = kFrameIntervalMs;
7697 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7698 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007699 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007700 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7701 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7702 VideoFrameType::kVideoFrameKey,
7703 VideoFrameType::kVideoFrameKey}));
7704
7705 // Disable top layer.
7706 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7707 config.simulcast_layers[number_layers - 1].active = false;
7708 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7709 sink_.SetNumExpectedLayers(number_layers - 1);
7710 timestamp_ms += kFrameIntervalMs;
7711 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7712 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007713 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007714 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7715 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7716 VideoFrameType::kVideoFrameDelta,
7717 VideoFrameType::kVideoFrameDelta}));
7718
7719 // Re-enable top layer.
7720 // Encoder should be re-initialized. Next frame should be key frame.
7721 config.simulcast_layers[number_layers - 1].active = true;
7722 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7723 sink_.SetNumExpectedLayers(number_layers);
7724 timestamp_ms += kFrameIntervalMs;
7725 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7726 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007727 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007728 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7729 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7730 VideoFrameType::kVideoFrameKey,
7731 VideoFrameType::kVideoFrameKey}));
7732
7733 // Top layer max rate change.
7734 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7735 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7736 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7737 sink_.SetNumExpectedLayers(number_layers);
7738 timestamp_ms += kFrameIntervalMs;
7739 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7740 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007741 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007742 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7743 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7744 VideoFrameType::kVideoFrameDelta,
7745 VideoFrameType::kVideoFrameDelta}));
7746
7747 // Top layer resolution change.
7748 // Encoder should be re-initialized. Next frame should be key frame.
7749 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7750 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7751 sink_.SetNumExpectedLayers(number_layers);
7752 timestamp_ms += kFrameIntervalMs;
7753 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7754 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007755 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007756 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7757 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7758 VideoFrameType::kVideoFrameKey,
7759 VideoFrameType::kVideoFrameKey}));
7760 video_stream_encoder_->Stop();
7761}
7762
Henrik Boström1124ed12021-02-25 10:30:39 +01007763TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7764 const int kFrameWidth = 1280;
7765 const int kFrameHeight = 720;
7766
7767 SetUp();
7768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007769 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007770
7771 // Capturing a frame should reconfigure the encoder and expose the encoder
7772 // resolution, which is the same as the input frame.
7773 int64_t timestamp_ms = kFrameIntervalMs;
7774 video_source_.IncomingCapturedFrame(
7775 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7776 WaitForEncodedFrame(timestamp_ms);
7777 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7778 EXPECT_THAT(video_source_.sink_wants().resolutions,
7779 ::testing::ElementsAreArray(
7780 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7781
7782 video_stream_encoder_->Stop();
7783}
7784
7785TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7786 // Pick downscale factors such that we never encode at full resolution - this
7787 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007788 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007789 // encoder should not ask for the frame resolution. This allows video frames
7790 // to have the appearence of one resolution but optimize its internal buffers
7791 // for what is actually encoded.
7792 const size_t kNumSimulcastLayers = 3u;
7793 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7794 const int kFrameWidth = 1280;
7795 const int kFrameHeight = 720;
7796 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7797 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7798 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7799 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7800 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7801 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7802
7803 VideoEncoderConfig config;
7804 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7805 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7806 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7807 config.simulcast_layers[i].active = true;
7808 }
7809 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007810 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007811 "VP8", /*max qp*/ 56, /*screencast*/ false,
7812 /*screenshare enabled*/ false);
7813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007814 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7815 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007816
7817 // Capture a frame with all layers active.
7818 int64_t timestamp_ms = kFrameIntervalMs;
7819 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7820 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7821 video_source_.IncomingCapturedFrame(
7822 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7823 WaitForEncodedFrame(timestamp_ms);
7824 // Expect encoded resolutions to match the expected simulcast layers.
7825 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7826 EXPECT_THAT(
7827 video_source_.sink_wants().resolutions,
7828 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7829
7830 // Capture a frame with one of the layers inactive.
7831 timestamp_ms += kFrameIntervalMs;
7832 config.simulcast_layers[2].active = false;
7833 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7834 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7835 video_source_.IncomingCapturedFrame(
7836 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7837 WaitForEncodedFrame(timestamp_ms);
7838
7839 // Expect encoded resolutions to match the expected simulcast layers.
7840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7841 EXPECT_THAT(video_source_.sink_wants().resolutions,
7842 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7843
7844 // Capture a frame with all but one layer turned off.
7845 timestamp_ms += kFrameIntervalMs;
7846 config.simulcast_layers[1].active = false;
7847 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7848 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7849 video_source_.IncomingCapturedFrame(
7850 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7851 WaitForEncodedFrame(timestamp_ms);
7852
7853 // Expect encoded resolutions to match the expected simulcast layers.
7854 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7855 EXPECT_THAT(video_source_.sink_wants().resolutions,
7856 ::testing::ElementsAreArray({kLayer0Size}));
7857
7858 video_stream_encoder_->Stop();
7859}
7860
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007861TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007862 ResetEncoder("VP8", 1, 1, 1, false);
7863
Niels Möller8b692902021-06-14 12:04:57 +02007864 // Force encoder reconfig.
7865 video_source_.IncomingCapturedFrame(
7866 CreateFrame(1, codec_width_, codec_height_));
7867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7868
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007869 // Set QP on encoded frame and pass the frame to encode complete callback.
7870 // Since QP is present QP parsing won't be triggered and the original value
7871 // should be kept.
7872 EncodedImage encoded_image;
7873 encoded_image.qp_ = 123;
7874 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7875 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7876 CodecSpecificInfo codec_info;
7877 codec_info.codecType = kVideoCodecVP8;
7878 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7879 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7880 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7881 video_stream_encoder_->Stop();
7882}
7883
7884TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007885 ResetEncoder("VP8", 1, 1, 1, false);
7886
Niels Möller8b692902021-06-14 12:04:57 +02007887 // Force encoder reconfig.
7888 video_source_.IncomingCapturedFrame(
7889 CreateFrame(1, codec_width_, codec_height_));
7890 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7891
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007892 // Pass an encoded frame without QP to encode complete callback. QP should be
7893 // parsed and set.
7894 EncodedImage encoded_image;
7895 encoded_image.qp_ = -1;
7896 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7897 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7898 CodecSpecificInfo codec_info;
7899 codec_info.codecType = kVideoCodecVP8;
7900 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7901 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7902 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7903 video_stream_encoder_->Stop();
7904}
7905
7906TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7907 webrtc::test::ScopedFieldTrials field_trials(
7908 "WebRTC-QpParsingKillSwitch/Enabled/");
7909
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007910 ResetEncoder("VP8", 1, 1, 1, false);
7911
Niels Möller8b692902021-06-14 12:04:57 +02007912 // Force encoder reconfig.
7913 video_source_.IncomingCapturedFrame(
7914 CreateFrame(1, codec_width_, codec_height_));
7915 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7916
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007917 EncodedImage encoded_image;
7918 encoded_image.qp_ = -1;
7919 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7920 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7921 CodecSpecificInfo codec_info;
7922 codec_info.codecType = kVideoCodecVP8;
7923 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7924 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7925 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7926 video_stream_encoder_->Stop();
7927}
7928
Sergey Silkind19e3b92021-03-16 10:05:30 +00007929TEST_F(VideoStreamEncoderTest,
7930 QualityScalingNotAllowed_QualityScalingDisabled) {
7931 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7932
7933 // Disable scaling settings in encoder info.
7934 fake_encoder_.SetQualityScaling(false);
7935 // Disable quality scaling in encoder config.
7936 video_encoder_config.is_quality_scaling_allowed = false;
7937 ConfigureEncoder(std::move(video_encoder_config));
7938
7939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007940 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007941
7942 test::FrameForwarder source;
7943 video_stream_encoder_->SetSource(
7944 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7945 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7946 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7947
7948 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7949 WaitForEncodedFrame(1);
7950 video_stream_encoder_->TriggerQualityLow();
7951 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7952
7953 video_stream_encoder_->Stop();
7954}
7955
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007956TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
7957 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7958
7959 // Disable scaling settings in encoder info.
7960 fake_encoder_.SetQualityScaling(false);
7961 // Set QP trusted in encoder info.
7962 fake_encoder_.SetIsQpTrusted(true);
7963 // Enable quality scaling in encoder config.
7964 video_encoder_config.is_quality_scaling_allowed = false;
7965 ConfigureEncoder(std::move(video_encoder_config));
7966
7967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007968 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007969
7970 test::FrameForwarder source;
7971 video_stream_encoder_->SetSource(
7972 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7973 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7974 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7975
7976 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7977 WaitForEncodedFrame(1);
7978 video_stream_encoder_->TriggerQualityLow();
7979 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7980
7981 video_stream_encoder_->Stop();
7982}
7983
Shuhai Pengf2707702021-09-29 17:19:44 +08007984TEST_F(VideoStreamEncoderTest,
7985 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
7986 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7987
7988 // Disable scaling settings in encoder info.
7989 fake_encoder_.SetQualityScaling(false);
7990 // Set QP trusted in encoder info.
7991 fake_encoder_.SetIsQpTrusted(true);
7992 // Enable quality scaling in encoder config.
7993 video_encoder_config.is_quality_scaling_allowed = false;
7994 ConfigureEncoder(std::move(video_encoder_config));
7995
7996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08007998
7999 test::FrameForwarder source;
8000 video_stream_encoder_->SetSource(
8001 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8002 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8003 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8004
8005 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8006 WaitForEncodedFrame(1);
8007 video_stream_encoder_->TriggerQualityLow();
8008 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8009
8010 video_stream_encoder_->Stop();
8011}
8012
8013TEST_F(VideoStreamEncoderTest,
8014 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8015 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8016
8017 // Disable scaling settings in encoder info.
8018 fake_encoder_.SetQualityScaling(false);
8019 // Set QP trusted in encoder info.
8020 fake_encoder_.SetIsQpTrusted(false);
8021 // Enable quality scaling in encoder config.
8022 video_encoder_config.is_quality_scaling_allowed = false;
8023 ConfigureEncoder(std::move(video_encoder_config));
8024
8025 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008026 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008027
8028 test::FrameForwarder source;
8029 video_stream_encoder_->SetSource(
8030 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8031 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8033
8034 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8035 WaitForEncodedFrame(1);
8036 video_stream_encoder_->TriggerQualityLow();
8037 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8038
8039 video_stream_encoder_->Stop();
8040}
8041
8042TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8043 // Set QP trusted in encoder info.
8044 fake_encoder_.SetIsQpTrusted(false);
8045
8046 const int MinEncBitrateKbps = 30;
8047 const int MaxEncBitrateKbps = 100;
8048 const int MinStartBitrateKbp = 50;
8049 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8050 /*frame_size_pixels=*/codec_width_ * codec_height_,
8051 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8052 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8053 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8054
8055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008056 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008057
8058 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8059
8060 VideoEncoderConfig video_encoder_config;
8061 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8062 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8063 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8064 MinEncBitrateKbps * 1000;
8065 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8066 kMaxPayloadLength);
8067
8068 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8069 WaitForEncodedFrame(1);
8070 EXPECT_EQ(
8071 MaxEncBitrateKbps,
8072 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8073 EXPECT_EQ(
8074 MinEncBitrateKbps,
8075 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8076
8077 video_stream_encoder_->Stop();
8078}
8079
8080TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8081 // Set QP trusted in encoder info.
8082 fake_encoder_.SetIsQpTrusted(false);
8083
8084 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8085 EncoderInfoSettings::
8086 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8087 codec_width_ * codec_height_,
8088 EncoderInfoSettings::
8089 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8090 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8091
8092 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8093 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8094 const int TargetEncBitrate = MaxEncBitrate;
8095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8096 DataRate::BitsPerSec(TargetEncBitrate),
8097 DataRate::BitsPerSec(TargetEncBitrate),
8098 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8099
8100 VideoEncoderConfig video_encoder_config;
8101 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8102 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8103 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8104 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8105 kMaxPayloadLength);
8106
8107 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8108 WaitForEncodedFrame(1);
8109 EXPECT_EQ(
8110 MaxEncBitrate / 1000,
8111 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8112 EXPECT_EQ(
8113 MinEncBitrate / 1000,
8114 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8115
8116 video_stream_encoder_->Stop();
8117}
8118
Sergey Silkind19e3b92021-03-16 10:05:30 +00008119#if !defined(WEBRTC_IOS)
8120// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8121// disabled by default on iOS.
8122TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8123 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8124
8125 // Disable scaling settings in encoder info.
8126 fake_encoder_.SetQualityScaling(false);
8127 // Enable quality scaling in encoder config.
8128 video_encoder_config.is_quality_scaling_allowed = true;
8129 ConfigureEncoder(std::move(video_encoder_config));
8130
8131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008132 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008133
8134 test::FrameForwarder source;
8135 video_stream_encoder_->SetSource(
8136 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8137 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8138 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8139
8140 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8141 WaitForEncodedFrame(1);
8142 video_stream_encoder_->TriggerQualityLow();
8143 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8144
8145 video_stream_encoder_->Stop();
8146}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008147
8148TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8149 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8150
8151 // Disable scaling settings in encoder info.
8152 fake_encoder_.SetQualityScaling(false);
8153 // Set QP trusted in encoder info.
8154 fake_encoder_.SetIsQpTrusted(true);
8155 // Enable quality scaling in encoder config.
8156 video_encoder_config.is_quality_scaling_allowed = true;
8157 ConfigureEncoder(std::move(video_encoder_config));
8158
8159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008160 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008161
8162 test::FrameForwarder source;
8163 video_stream_encoder_->SetSource(
8164 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8165 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8166 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8167
8168 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8169 WaitForEncodedFrame(1);
8170 video_stream_encoder_->TriggerQualityLow();
8171 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8172
8173 video_stream_encoder_->Stop();
8174}
Shuhai Pengf2707702021-09-29 17:19:44 +08008175
8176TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8177 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8178
8179 // Disable scaling settings in encoder info.
8180 fake_encoder_.SetQualityScaling(false);
8181 // Set QP not trusted in encoder info.
8182 fake_encoder_.SetIsQpTrusted(false);
8183 // Enable quality scaling in encoder config.
8184 video_encoder_config.is_quality_scaling_allowed = true;
8185 ConfigureEncoder(std::move(video_encoder_config));
8186
8187 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008188 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008189
8190 test::FrameForwarder source;
8191 video_stream_encoder_->SetSource(
8192 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8193 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8195
8196 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8197 WaitForEncodedFrame(1);
8198 video_stream_encoder_->TriggerQualityLow();
8199 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8200 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8201 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8202
8203 video_stream_encoder_->Stop();
8204}
8205
8206TEST_F(VideoStreamEncoderTest,
8207 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8208 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8209
8210 // Disable scaling settings in encoder info.
8211 fake_encoder_.SetQualityScaling(false);
8212 // Set QP trusted in encoder info.
8213 fake_encoder_.SetIsQpTrusted(true);
8214 // Enable quality scaling in encoder config.
8215 video_encoder_config.is_quality_scaling_allowed = true;
8216 ConfigureEncoder(std::move(video_encoder_config));
8217
8218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008219 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008220
8221 test::FrameForwarder source;
8222 video_stream_encoder_->SetSource(
8223 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8224 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8225 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8226
8227 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8228 WaitForEncodedFrame(1);
8229 video_stream_encoder_->TriggerQualityLow();
8230 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8231 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8232
8233 video_stream_encoder_->Stop();
8234}
8235
8236TEST_F(VideoStreamEncoderTest,
8237 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8238 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8239
8240 // Disable scaling settings in encoder info.
8241 fake_encoder_.SetQualityScaling(false);
8242 // Set QP trusted in encoder info.
8243 fake_encoder_.SetIsQpTrusted(false);
8244 // Enable quality scaling in encoder config.
8245 video_encoder_config.is_quality_scaling_allowed = true;
8246 ConfigureEncoder(std::move(video_encoder_config));
8247
8248 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008249 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008250
8251 test::FrameForwarder source;
8252 video_stream_encoder_->SetSource(
8253 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8254 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8255 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8256
8257 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8258 WaitForEncodedFrame(1);
8259 video_stream_encoder_->TriggerQualityLow();
8260 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8261
8262 video_stream_encoder_->Stop();
8263}
8264
Sergey Silkind19e3b92021-03-16 10:05:30 +00008265#endif
8266
Henrik Boström56db9ff2021-03-24 09:06:45 +01008267// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8268class VideoStreamEncoderWithRealEncoderTest
8269 : public VideoStreamEncoderTest,
8270 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8271 public:
8272 VideoStreamEncoderWithRealEncoderTest()
8273 : VideoStreamEncoderTest(),
8274 codec_type_(std::get<0>(GetParam())),
8275 allow_i420_conversion_(std::get<1>(GetParam())) {}
8276
8277 void SetUp() override {
8278 VideoStreamEncoderTest::SetUp();
8279 std::unique_ptr<VideoEncoder> encoder;
8280 switch (codec_type_) {
8281 case kVideoCodecVP8:
8282 encoder = VP8Encoder::Create();
8283 break;
8284 case kVideoCodecVP9:
8285 encoder = VP9Encoder::Create();
8286 break;
8287 case kVideoCodecAV1:
8288 encoder = CreateLibaomAv1Encoder();
8289 break;
8290 case kVideoCodecH264:
8291 encoder =
8292 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8293 break;
8294 case kVideoCodecMultiplex:
8295 mock_encoder_factory_for_multiplex_ =
8296 std::make_unique<MockVideoEncoderFactory>();
8297 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8298 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8299 .WillRepeatedly([] { return VP8Encoder::Create(); });
8300 encoder = std::make_unique<MultiplexEncoderAdapter>(
8301 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8302 false);
8303 break;
8304 default:
8305 RTC_NOTREACHED();
8306 }
8307 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8308 }
8309
8310 void TearDown() override {
8311 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008312 // Ensure `video_stream_encoder_` is destroyed before
8313 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008314 video_stream_encoder_.reset();
8315 VideoStreamEncoderTest::TearDown();
8316 }
8317
8318 protected:
8319 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8320 std::unique_ptr<VideoEncoder> encoder) {
8321 // Configure VSE to use the encoder.
8322 encoder_ = std::move(encoder);
8323 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8324 encoder_.get(), &encoder_selector_);
8325 video_send_config_.encoder_settings.encoder_factory =
8326 encoder_proxy_factory_.get();
8327 VideoEncoderConfig video_encoder_config;
8328 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8329 video_encoder_config_ = video_encoder_config.Copy();
8330 ConfigureEncoder(video_encoder_config_.Copy());
8331
8332 // Set bitrate to ensure frame is not dropped.
8333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008335 }
8336
8337 const VideoCodecType codec_type_;
8338 const bool allow_i420_conversion_;
8339 NiceMock<MockEncoderSelector> encoder_selector_;
8340 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8341 std::unique_ptr<VideoEncoder> encoder_;
8342 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8343};
8344
8345TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8346 auto native_i420_frame = test::CreateMappableNativeFrame(
8347 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8348 video_source_.IncomingCapturedFrame(native_i420_frame);
8349 WaitForEncodedFrame(codec_width_, codec_height_);
8350
8351 auto mappable_native_buffer =
8352 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8353 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8354 mappable_native_buffer->GetMappedFramedBuffers();
8355 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8356 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8357 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8358 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8359}
8360
8361TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8362 auto native_nv12_frame = test::CreateMappableNativeFrame(
8363 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8364 video_source_.IncomingCapturedFrame(native_nv12_frame);
8365 WaitForEncodedFrame(codec_width_, codec_height_);
8366
8367 auto mappable_native_buffer =
8368 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8369 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8370 mappable_native_buffer->GetMappedFramedBuffers();
8371 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8372 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8373 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8374 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8375
8376 if (!allow_i420_conversion_) {
8377 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8378 }
8379}
8380
Erik Språng7444b192021-06-02 14:02:13 +02008381TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8382 if (codec_type_ == kVideoCodecMultiplex) {
8383 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8384 return;
8385 }
8386
8387 const size_t kNumSpatialLayers = 3u;
8388 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8389 const int kFrameWidth = 1280;
8390 const int kFrameHeight = 720;
8391 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8392 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8393 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8394 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8395 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8396 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8397
8398 VideoEncoderConfig config;
8399 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8400 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008401 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008402 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8403 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8404 vp9_settings.numberOfTemporalLayers = 3;
8405 vp9_settings.automaticResizeOn = false;
8406 config.encoder_specific_settings =
8407 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8408 vp9_settings);
8409 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8410 /*fps=*/30.0,
8411 /*first_active_layer=*/0,
8412 /*num_spatial_layers=*/3,
8413 /*num_temporal_layers=*/3,
8414 /*is_screenshare=*/false);
8415 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8416 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008417 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008418 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8419 /*fps=*/30.0,
8420 /*first_active_layer=*/0,
8421 /*num_spatial_layers=*/3,
8422 /*num_temporal_layers=*/3,
8423 /*is_screenshare=*/false);
8424 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8425 } else {
8426 // Simulcast for VP8/H264.
8427 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8428 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8429 config.simulcast_layers[i].scale_resolution_down_by =
8430 kDownscaleFactors[i];
8431 config.simulcast_layers[i].active = true;
8432 }
8433 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8434 // Turn off frame dropping to prevent flakiness.
8435 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8436 h264_settings.frameDroppingOn = false;
8437 config.encoder_specific_settings = rtc::make_ref_counted<
8438 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8439 }
8440 }
8441
8442 auto set_layer_active = [&](int layer_idx, bool active) {
8443 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8444 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8445 config.spatial_layers[layer_idx].active = active;
8446 } else {
8447 config.simulcast_layers[layer_idx].active = active;
8448 }
8449 };
8450
8451 config.video_stream_factory =
8452 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8453 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8454 /*screencast*/ false,
8455 /*screenshare enabled*/ false);
8456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008457 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8458 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008459
8460 // Capture a frame with all layers active.
8461 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8462 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8463 int64_t timestamp_ms = kFrameIntervalMs;
8464 video_source_.IncomingCapturedFrame(
8465 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8466
8467 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8468 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8469
8470 // Capture a frame with one of the layers inactive.
8471 set_layer_active(2, false);
8472 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8473 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8474 timestamp_ms += kFrameIntervalMs;
8475 video_source_.IncomingCapturedFrame(
8476 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8477 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8478
8479 // New target bitrates signaled based on lower resolution.
8480 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8482 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8483 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8484
8485 // Re-enable the top layer.
8486 set_layer_active(2, true);
8487 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8488 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8489 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8490
8491 // Bitrate target adjusted back up to enable HD layer...
8492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8493 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8494 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8495 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8496
8497 // ...then add a new frame.
8498 timestamp_ms += kFrameIntervalMs;
8499 video_source_.IncomingCapturedFrame(
8500 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8501 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8502 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8503
8504 video_stream_encoder_->Stop();
8505}
8506
Henrik Boström56db9ff2021-03-24 09:06:45 +01008507std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8508 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8509 VideoCodecType codec_type = std::get<0>(info.param);
8510 bool allow_i420_conversion = std::get<1>(info.param);
8511 std::string str;
8512 switch (codec_type) {
8513 case kVideoCodecGeneric:
8514 str = "Generic";
8515 break;
8516 case kVideoCodecVP8:
8517 str = "VP8";
8518 break;
8519 case kVideoCodecVP9:
8520 str = "VP9";
8521 break;
8522 case kVideoCodecAV1:
8523 str = "AV1";
8524 break;
8525 case kVideoCodecH264:
8526 str = "H264";
8527 break;
8528 case kVideoCodecMultiplex:
8529 str = "Multiplex";
8530 break;
8531 default:
8532 RTC_NOTREACHED();
8533 }
8534 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8535 return str;
8536}
8537
8538constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8539 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8540constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8541 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8542constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8543 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8544constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8545 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8546#if defined(WEBRTC_USE_H264)
8547constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8548 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8549
8550// The windows compiler does not tolerate #if statements inside the
8551// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8552// and without H264).
8553INSTANTIATE_TEST_SUITE_P(
8554 All,
8555 VideoStreamEncoderWithRealEncoderTest,
8556 ::testing::Values(kVP8DisallowConversion,
8557 kVP9DisallowConversion,
8558 kAV1AllowConversion,
8559 kMultiplexDisallowConversion,
8560 kH264AllowConversion),
8561 TestParametersVideoCodecAndAllowI420ConversionToString);
8562#else
8563INSTANTIATE_TEST_SUITE_P(
8564 All,
8565 VideoStreamEncoderWithRealEncoderTest,
8566 ::testing::Values(kVP8DisallowConversion,
8567 kVP9DisallowConversion,
8568 kAV1AllowConversion,
8569 kMultiplexDisallowConversion),
8570 TestParametersVideoCodecAndAllowI420ConversionToString);
8571#endif
8572
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008573class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8574 protected:
8575 void RunTest(const std::vector<VideoStream>& configs,
8576 const int expected_num_init_encode) {
8577 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008578 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008579 InsertFrameAndWaitForEncoded();
8580 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8581 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008582 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8583 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008584
8585 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8586 ConfigureEncoder(configs[1]);
8587 InsertFrameAndWaitForEncoded();
8588 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8589 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008590 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008591 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008592 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008593
8594 video_stream_encoder_->Stop();
8595 }
8596
8597 void ConfigureEncoder(const VideoStream& stream) {
8598 VideoEncoderConfig config;
8599 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8600 config.max_bitrate_bps = stream.max_bitrate_bps;
8601 config.simulcast_layers[0] = stream;
8602 config.video_stream_factory =
8603 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8604 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8605 /*conference_mode=*/false);
8606 video_stream_encoder_->ConfigureEncoder(std::move(config),
8607 kMaxPayloadLength);
8608 }
8609
8610 void OnBitrateUpdated(DataRate bitrate) {
8611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8612 bitrate, bitrate, bitrate, 0, 0, 0);
8613 }
8614
8615 void InsertFrameAndWaitForEncoded() {
8616 timestamp_ms_ += kFrameIntervalMs;
8617 video_source_.IncomingCapturedFrame(
8618 CreateFrame(timestamp_ms_, kWidth, kHeight));
8619 sink_.WaitForEncodedFrame(timestamp_ms_);
8620 }
8621
8622 void ExpectEqual(const VideoCodec& actual,
8623 const VideoStream& expected) const {
8624 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8625 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8626 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8627 static_cast<unsigned int>(expected.min_bitrate_bps));
8628 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8629 static_cast<unsigned int>(expected.max_bitrate_bps));
8630 EXPECT_EQ(actual.simulcastStream[0].width,
8631 kWidth / expected.scale_resolution_down_by);
8632 EXPECT_EQ(actual.simulcastStream[0].height,
8633 kHeight / expected.scale_resolution_down_by);
8634 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8635 expected.num_temporal_layers);
8636 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8637 }
8638
8639 VideoStream DefaultConfig() const {
8640 VideoStream stream;
8641 stream.max_framerate = 25;
8642 stream.min_bitrate_bps = 35000;
8643 stream.max_bitrate_bps = 900000;
8644 stream.scale_resolution_down_by = 1.0;
8645 stream.num_temporal_layers = 1;
8646 stream.bitrate_priority = 1.0;
8647 stream.scalability_mode = "";
8648 return stream;
8649 }
8650
8651 const int kWidth = 640;
8652 const int kHeight = 360;
8653 int64_t timestamp_ms_ = 0;
8654};
8655
8656TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8657 VideoStream config1 = DefaultConfig();
8658 VideoStream config2 = config1;
8659 config2.max_framerate++;
8660
8661 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8662}
8663
8664TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8665 VideoStream config1 = DefaultConfig();
8666 VideoStream config2 = config1;
8667 config2.min_bitrate_bps += 10000;
8668
8669 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8670}
8671
8672TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8673 VideoStream config1 = DefaultConfig();
8674 VideoStream config2 = config1;
8675 config2.max_bitrate_bps += 100000;
8676
8677 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8678}
8679
8680TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8681 VideoStream config1 = DefaultConfig();
8682 VideoStream config2 = config1;
8683 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8684
8685 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8686}
8687
8688TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8689 VideoStream config1 = DefaultConfig();
8690 VideoStream config2 = config1;
8691 config2.scale_resolution_down_by *= 2;
8692
8693 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8694}
8695
8696TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8697 VideoStream config1 = DefaultConfig();
8698 VideoStream config2 = config1;
8699 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8700
8701 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8702}
8703
8704TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8705 VideoStream config1 = DefaultConfig();
8706 VideoStream config2 = config1;
8707 config2.scalability_mode = "L1T2";
8708
8709 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8710}
8711
perkj26091b12016-09-01 01:17:40 -07008712} // namespace webrtc