blob: cbfd93e9e23f71b5bbf2082d338a9435807c632f [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020045#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020046#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010047#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020048#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010049#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020050#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020053#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010054#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010062#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020063#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020064#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070066
67namespace webrtc {
68
sprang57c2fff2017-01-16 06:24:02 -080069using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020071using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020072using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010081using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080083
perkj803d97f2016-11-01 11:45:46 -070084namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080091const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010092const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020093const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010094const uint32_t kSimulcastTargetBitrateBps = 3150000;
95const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080096const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070097const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020099const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800104
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200105uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 11:45:46 -0700115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 09:06:45 +0100129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150 }
Noah Richards51db4212019-06-12 06:59:12 -0700151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 09:15:13 +0200163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 10:27:48 +0100194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 09:04:13 +0200215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 18:49:07 +0200217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219 private:
Markus Handella3765182020-07-08 13:13:32 +0200220 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100223};
224
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200227 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 08:27:51 -0700343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700344 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100358 task_queue_factory,
359 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200360 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200361 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200362 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200363 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200364 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200365 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200366 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200367 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100368 }
perkj803d97f2016-11-01 11:45:46 -0700369
Henrik Boström381d1092020-05-12 18:49:07 +0200370 void SetSourceAndWaitForRestrictionsUpdated(
371 rtc::VideoSourceInterface<VideoFrame>* source,
372 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 FakeVideoSourceRestrictionsListener listener;
374 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200375 SetSource(source, degradation_preference);
376 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200377 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200378 }
379
380 void SetSourceAndWaitForFramerateUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
383 overuse_detector_proxy_->framerate_updated_event()->Reset();
384 SetSource(source, degradation_preference);
385 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
386 }
387
388 void OnBitrateUpdatedAndWaitForManagedResources(
389 DataRate target_bitrate,
390 DataRate stable_target_bitrate,
391 DataRate link_allocation,
392 uint8_t fraction_lost,
393 int64_t round_trip_time_ms,
394 double cwnd_reduce_ratio) {
395 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
396 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
397 // Bitrate is updated on the encoder queue.
398 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200399 }
400
kthelgason2fc52542017-03-03 00:24:41 -0800401 // This is used as a synchronisation mechanism, to make sure that the
402 // encoder queue is not blocked before we start sending it frames.
403 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100404 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200405 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800406 ASSERT_TRUE(event.Wait(5000));
407 }
408
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200419
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 void TriggerCpuUnderuse() {
421 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200422 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200423 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 event.Set();
425 });
426 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200427 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200428 }
kthelgason876222f2016-11-29 01:44:11 -0800429
Henrik Boström91aa7322020-04-28 12:24:33 +0200430 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200431 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200433 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200434 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200435 event.Set();
436 });
437 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200438 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200440 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200443 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 }
449
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100451 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200452 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
453 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700455};
456
Noah Richards51db4212019-06-12 06:59:12 -0700457// Simulates simulcast behavior and makes highest stream resolutions divisible
458// by 4.
459class CroppingVideoStreamFactory
460 : public VideoEncoderConfig::VideoStreamFactoryInterface {
461 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200462 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700463
464 private:
465 std::vector<VideoStream> CreateEncoderStreams(
466 int width,
467 int height,
468 const VideoEncoderConfig& encoder_config) override {
469 std::vector<VideoStream> streams = test::CreateVideoStreams(
470 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700471 return streams;
472 }
Noah Richards51db4212019-06-12 06:59:12 -0700473};
474
sprangb1ca0732017-02-01 08:38:12 -0800475class AdaptingFrameForwarder : public test::FrameForwarder {
476 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 explicit AdaptingFrameForwarder(TimeController* time_controller)
478 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700479 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800480
481 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200482 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800483 adaptation_enabled_ = enabled;
484 }
485
asaperssonfab67072017-04-04 05:51:49 -0700486 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800488 return adaptation_enabled_;
489 }
490
Henrik Boström1124ed12021-02-25 10:30:39 +0100491 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
492 // the resolution or frame rate was different than it is currently. If
493 // something else is modified, such as encoder resolutions, but the resolution
494 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700495 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200496 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700497 return last_wants_;
498 }
499
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200500 absl::optional<int> last_sent_width() const { return last_width_; }
501 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800502
sprangb1ca0732017-02-01 08:38:12 -0800503 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200504 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
505 time_controller_->AdvanceTime(TimeDelta::Millis(0));
506
sprangb1ca0732017-02-01 08:38:12 -0800507 int cropped_width = 0;
508 int cropped_height = 0;
509 int out_width = 0;
510 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700511 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
513 << "w=" << video_frame.width()
514 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700515 if (adapter_.AdaptFrameResolution(
516 video_frame.width(), video_frame.height(),
517 video_frame.timestamp_us() * 1000, &cropped_width,
518 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame adapted_frame =
520 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200521 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100522 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200523 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-02 23:53:04 -0700534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700540 }
sprangb1ca0732017-02-01 08:38:12 -0800541 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800546 }
547 }
548
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200549 void OnOutputFormatRequest(int width, int height) {
550 absl::optional<std::pair<int, int>> target_aspect_ratio =
551 std::make_pair(width, height);
552 absl::optional<int> max_pixel_count = width * height;
553 absl::optional<int> max_fps;
554 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
555 max_fps);
556 }
557
sprangb1ca0732017-02-01 08:38:12 -0800558 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
559 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200560 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100561 rtc::VideoSinkWants prev_wants = sink_wants_locked();
562 bool did_adapt =
563 prev_wants.max_pixel_count != wants.max_pixel_count ||
564 prev_wants.target_pixel_count != wants.target_pixel_count ||
565 prev_wants.max_framerate_fps != wants.max_framerate_fps;
566 if (did_adapt) {
567 last_wants_ = prev_wants;
568 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100569 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200570 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800571 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200572
573 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800574 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200575 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
576 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200577 absl::optional<int> last_width_;
578 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800579};
sprangc5d62e22017-04-02 23:53:04 -0700580
Niels Möller213618e2018-07-24 09:29:58 +0200581// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700582class MockableSendStatisticsProxy : public SendStatisticsProxy {
583 public:
584 MockableSendStatisticsProxy(Clock* clock,
585 const VideoSendStream::Config& config,
586 VideoEncoderConfig::ContentType content_type)
587 : SendStatisticsProxy(clock, config, content_type) {}
588
589 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200590 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700591 if (mock_stats_)
592 return *mock_stats_;
593 return SendStatisticsProxy::GetStats();
594 }
595
Niels Möller213618e2018-07-24 09:29:58 +0200596 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200597 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200598 if (mock_stats_)
599 return mock_stats_->input_frame_rate;
600 return SendStatisticsProxy::GetInputFrameRate();
601 }
sprangc5d62e22017-04-02 23:53:04 -0700602 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200603 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700604 mock_stats_.emplace(stats);
605 }
606
607 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200608 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700609 mock_stats_.reset();
610 }
611
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200612 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
613 on_frame_dropped_ = std::move(callback);
614 }
615
sprangc5d62e22017-04-02 23:53:04 -0700616 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 void OnFrameDropped(DropReason reason) override {
618 SendStatisticsProxy::OnFrameDropped(reason);
619 if (on_frame_dropped_)
620 on_frame_dropped_(reason);
621 }
622
Markus Handella3765182020-07-08 13:13:32 +0200623 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200624 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200625 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700626};
627
philipel9b058032020-02-10 11:30:00 +0100628class MockEncoderSelector
629 : public VideoEncoderFactory::EncoderSelectorInterface {
630 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200631 MOCK_METHOD(void,
632 OnCurrentEncoder,
633 (const SdpVideoFormat& format),
634 (override));
635 MOCK_METHOD(absl::optional<SdpVideoFormat>,
636 OnAvailableBitrate,
637 (const DataRate& rate),
638 (override));
639 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100640};
641
perkj803d97f2016-11-01 11:45:46 -0700642} // namespace
643
mflodmancc3d4422017-08-03 08:27:51 -0700644class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700645 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200646 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700647
mflodmancc3d4422017-08-03 08:27:51 -0700648 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700649 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700650 codec_width_(320),
651 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200652 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200654 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700655 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200656 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700657 video_send_config_,
658 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200659 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700660
661 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700662 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700663 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200664 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800665 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200666 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200667 video_send_config_.rtp.payload_name = "FAKE";
668 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700669
Per512ecb32016-09-23 15:52:06 +0200670 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200671 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200672 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
673 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
674 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100675 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700676
Niels Möllerf1338562018-04-26 09:51:47 +0200677 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800678 }
679
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100680 void ConfigureEncoder(
681 VideoEncoderConfig video_encoder_config,
682 VideoStreamEncoder::BitrateAllocationCallbackType
683 allocation_callback_type =
684 VideoStreamEncoder::BitrateAllocationCallbackType::
685 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700686 if (video_stream_encoder_)
687 video_stream_encoder_->Stop();
688 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200689 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100690 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700691 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700693 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700694 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200696 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800698 }
699
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100700 void ResetEncoder(const std::string& payload_name,
701 size_t num_streams,
702 size_t num_temporal_layers,
703 unsigned char num_spatial_layers,
704 bool screenshare,
705 VideoStreamEncoder::BitrateAllocationCallbackType
706 allocation_callback_type =
707 VideoStreamEncoder::BitrateAllocationCallbackType::
708 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200709 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800710
711 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200712 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
713 num_streams, &video_encoder_config);
714 for (auto& layer : video_encoder_config.simulcast_layers) {
715 layer.num_temporal_layers = num_temporal_layers;
716 layer.max_framerate = kDefaultFramerate;
717 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100718 video_encoder_config.max_bitrate_bps =
719 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700720 video_encoder_config.content_type =
721 screenshare ? VideoEncoderConfig::ContentType::kScreen
722 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700723 if (payload_name == "VP9") {
724 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
725 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200726 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200728 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
729 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700730 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100731 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700732 }
733
sprang57c2fff2017-01-16 06:24:02 -0800734 VideoFrame CreateFrame(int64_t ntp_time_ms,
735 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200736 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200737 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200738 destruction_event, codec_width_, codec_height_))
739 .set_ntp_time_ms(ntp_time_ms)
740 .set_timestamp_ms(99)
741 .set_rotation(kVideoRotation_0)
742 .build();
perkj26091b12016-09-01 01:17:40 -0700743 }
744
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100745 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200748 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200749 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200750 destruction_event, codec_width_, codec_height_))
751 .set_ntp_time_ms(ntp_time_ms)
752 .set_timestamp_ms(99)
753 .set_rotation(kVideoRotation_0)
754 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
755 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100756 }
757
sprang57c2fff2017-01-16 06:24:02 -0800758 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200759 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
760 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200761 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200762 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200763 .set_ntp_time_ms(ntp_time_ms)
764 .set_timestamp_ms(ntp_time_ms)
765 .set_rotation(kVideoRotation_0)
766 .build();
perkj803d97f2016-11-01 11:45:46 -0700767 }
768
Evan Shrubsole895556e2020-10-05 09:15:13 +0200769 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200770 return VideoFrame::Builder()
771 .set_video_frame_buffer(NV12Buffer::Create(width, height))
772 .set_ntp_time_ms(ntp_time_ms)
773 .set_timestamp_ms(ntp_time_ms)
774 .set_rotation(kVideoRotation_0)
775 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200776 }
777
Noah Richards51db4212019-06-12 06:59:12 -0700778 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200782 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200783 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200784 destruction_event, width, height))
785 .set_ntp_time_ms(ntp_time_ms)
786 .set_timestamp_ms(99)
787 .set_rotation(kVideoRotation_0)
788 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700789 }
790
Evan Shrubsole895556e2020-10-05 09:15:13 +0200791 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
792 rtc::Event* destruction_event,
793 int width,
794 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200795 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200796 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200797 destruction_event, width, height))
798 .set_ntp_time_ms(ntp_time_ms)
799 .set_timestamp_ms(99)
800 .set_rotation(kVideoRotation_0)
801 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200802 }
803
Noah Richards51db4212019-06-12 06:59:12 -0700804 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
805 rtc::Event* destruction_event) const {
806 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
807 codec_height_);
808 }
809
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100810 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100812 DataRate::BitsPerSec(kTargetBitrateBps),
813 DataRate::BitsPerSec(kTargetBitrateBps),
814 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100815
816 video_source_.IncomingCapturedFrame(
817 CreateFrame(1, codec_width_, codec_height_));
818 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200819 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100820 }
821
sprang4847ae62017-06-27 07:06:52 -0700822 void WaitForEncodedFrame(int64_t expected_ntp_time) {
823 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700825 }
826
827 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
828 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200829 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700830 return ok;
831 }
832
833 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
834 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700836 }
837
838 void ExpectDroppedFrame() {
839 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200840 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700841 }
842
843 bool WaitForFrame(int64_t timeout_ms) {
844 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200845 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700846 return ok;
847 }
848
perkj26091b12016-09-01 01:17:40 -0700849 class TestEncoder : public test::FakeEncoder {
850 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200851 explicit TestEncoder(TimeController* time_controller)
852 : FakeEncoder(time_controller->GetClock()),
853 time_controller_(time_controller) {
854 RTC_DCHECK(time_controller_);
855 }
perkj26091b12016-09-01 01:17:40 -0700856
asaperssonfab67072017-04-04 05:51:49 -0700857 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200858 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700859 return config_;
860 }
861
862 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200863 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700864 block_next_encode_ = true;
865 }
866
Erik Språngaed30702018-11-05 12:57:17 +0100867 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200868 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200869 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100871 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100872 info.scaling_settings = VideoEncoder::ScalingSettings(
873 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100874 }
875 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100876 for (int i = 0; i < kMaxSpatialLayers; ++i) {
877 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100878 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100879 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100880 for (int tid = 0; tid < num_layers; ++tid)
881 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100882 }
883 }
Erik Språngaed30702018-11-05 12:57:17 +0100884 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200885
886 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100887 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200888 info.apply_alignment_to_all_simulcast_layers =
889 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200890 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100891 return info;
kthelgason876222f2016-11-29 01:44:11 -0800892 }
893
Erik Språngb7cb7b52019-02-26 15:52:33 +0100894 int32_t RegisterEncodeCompleteCallback(
895 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200896 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100897 encoded_image_callback_ = callback;
898 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
899 }
900
perkjfa10b552016-10-02 23:45:26 -0700901 void ContinueEncode() { continue_encode_event_.Set(); }
902
903 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
904 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700906 EXPECT_EQ(timestamp_, timestamp);
907 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
908 }
909
kthelgason2fc52542017-03-03 00:24:41 -0800910 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200911 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800912 quality_scaling_ = b;
913 }
kthelgasonad9010c2017-02-14 00:46:51 -0800914
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100915 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200916 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100917 requested_resolution_alignment_ = requested_resolution_alignment;
918 }
919
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200920 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
921 MutexLock lock(&local_mutex_);
922 apply_alignment_to_all_simulcast_layers_ = b;
923 }
924
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100925 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100927 is_hardware_accelerated_ = is_hardware_accelerated;
928 }
929
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100930 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
931 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200932 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100933 temporal_layers_supported_[spatial_idx] = supported;
934 }
935
Sergey Silkin6456e352019-07-08 17:56:40 +0200936 void SetResolutionBitrateLimits(
937 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200938 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200939 resolution_bitrate_limits_ = thresholds;
940 }
941
sprangfe627f32017-03-29 08:24:59 -0700942 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200943 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700944 force_init_encode_failed_ = force_failure;
945 }
946
Niels Möller6bb5ab92019-01-11 11:11:10 +0100947 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200948 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100949 rate_factor_ = rate_factor;
950 }
951
Erik Språngd7329ca2019-02-21 21:19:53 +0100952 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200953 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100954 return last_framerate_;
955 }
956
Erik Språngd7329ca2019-02-21 21:19:53 +0100957 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200958 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100959 return last_update_rect_;
960 }
961
Niels Möller87e2d782019-03-07 10:18:23 +0100962 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 return last_frame_types_;
965 }
966
967 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100968 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100969 keyframe ? VideoFrameType::kVideoFrameKey
970 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100971 {
Markus Handella3765182020-07-08 13:13:32 +0200972 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100973 last_frame_types_ = frame_type;
974 }
Niels Möllerb859b322019-03-07 12:40:01 +0100975 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100976 }
977
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100978 void InjectEncodedImage(const EncodedImage& image,
979 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100981 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100982 }
983
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200984 void SetEncodedImageData(
985 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200986 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200987 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200988 }
989
Erik Språngd7329ca2019-02-21 21:19:53 +0100990 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200991 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100992 expect_null_frame_ = true;
993 }
994
Erik Språng5056af02019-09-02 15:53:11 +0200995 absl::optional<VideoEncoder::RateControlParameters>
996 GetAndResetLastRateControlSettings() {
997 auto settings = last_rate_control_settings_;
998 last_rate_control_settings_.reset();
999 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001000 }
1001
Henrik Boström56db9ff2021-03-24 09:06:45 +01001002 int GetLastInputWidth() const {
1003 MutexLock lock(&local_mutex_);
1004 return last_input_width_;
1005 }
1006
1007 int GetLastInputHeight() const {
1008 MutexLock lock(&local_mutex_);
1009 return last_input_height_;
1010 }
1011
Evan Shrubsole895556e2020-10-05 09:15:13 +02001012 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1013 MutexLock lock(&local_mutex_);
1014 return last_input_pixel_format_;
1015 }
1016
Sergey Silkin5ee69672019-07-02 14:18:34 +02001017 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +02001018 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001019 return num_encoder_initializations_;
1020 }
1021
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001022 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001023 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001024 return num_set_rates_;
1025 }
1026
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001027 VideoCodec video_codec() const {
1028 MutexLock lock(&local_mutex_);
1029 return video_codec_;
1030 }
1031
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001032 void SetPreferredPixelFormats(
1033 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1034 pixel_formats) {
1035 MutexLock lock(&local_mutex_);
1036 preferred_pixel_formats_ = std::move(pixel_formats);
1037 }
1038
perkjfa10b552016-10-02 23:45:26 -07001039 private:
perkj26091b12016-09-01 01:17:40 -07001040 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001041 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001042 bool block_encode;
1043 {
Markus Handella3765182020-07-08 13:13:32 +02001044 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001045 if (expect_null_frame_) {
1046 EXPECT_EQ(input_image.timestamp(), 0u);
1047 EXPECT_EQ(input_image.width(), 1);
1048 last_frame_types_ = *frame_types;
1049 expect_null_frame_ = false;
1050 } else {
1051 EXPECT_GT(input_image.timestamp(), timestamp_);
1052 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1053 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1054 }
perkj26091b12016-09-01 01:17:40 -07001055
1056 timestamp_ = input_image.timestamp();
1057 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001058 last_input_width_ = input_image.width();
1059 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001060 block_encode = block_next_encode_;
1061 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001062 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001063 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001064 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001065 }
Niels Möllerb859b322019-03-07 12:40:01 +01001066 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001067 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001068 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001069
perkj26091b12016-09-01 01:17:40 -07001070 return result;
1071 }
1072
Niels Möller08ae7ce2020-09-23 15:58:12 +02001073 CodecSpecificInfo EncodeHook(
1074 EncodedImage& encoded_image,
1075 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001076 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001077 {
1078 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001079 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001080 }
1081 MutexLock lock(&local_mutex_);
1082 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001083 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001084 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001085 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001086 }
1087
sprangfe627f32017-03-29 08:24:59 -07001088 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001089 const Settings& settings) override {
1090 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001091
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001093 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001094
1095 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001096 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001097
Erik Språng82fad3d2018-03-21 09:57:23 +01001098 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001099 // Simulate setting up temporal layers, in order to validate the life
1100 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001101 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001102 frame_buffer_controller_ =
1103 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001104 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001105 if (force_init_encode_failed_) {
1106 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001107 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001108 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001109
Erik Språngb7cb7b52019-02-26 15:52:33 +01001110 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001111 return res;
1112 }
1113
Erik Språngb7cb7b52019-02-26 15:52:33 +01001114 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001115 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001116 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1117 initialized_ = EncoderState::kUninitialized;
1118 return FakeEncoder::Release();
1119 }
1120
Erik Språng16cb8f52019-04-12 13:59:09 +02001121 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001123 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001124 VideoBitrateAllocation adjusted_rate_allocation;
1125 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1126 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001127 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001128 adjusted_rate_allocation.SetBitrate(
1129 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001130 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001131 rate_factor_));
1132 }
1133 }
1134 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001135 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001136 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001137 RateControlParameters adjusted_paramters = parameters;
1138 adjusted_paramters.bitrate = adjusted_rate_allocation;
1139 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001140 }
1141
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001142 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001143 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001144 enum class EncoderState {
1145 kUninitialized,
1146 kInitializationFailed,
1147 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001148 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1149 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001150 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001151 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1152 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1153 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1154 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1155 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1156 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001157 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1158 false;
Markus Handella3765182020-07-08 13:13:32 +02001159 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001160 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1161 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001162 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001163 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001164 absl::optional<bool>
1165 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001166 local_mutex_);
1167 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1168 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1169 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001170 absl::optional<VideoEncoder::RateControlParameters>
1171 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001172 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1173 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001174 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001175 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001176 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1177 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001178 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001179 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001180 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001181 RTC_GUARDED_BY(local_mutex_);
1182 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001183 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001184 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1185 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001186 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1187 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001188 };
1189
mflodmancc3d4422017-08-03 08:27:51 -07001190 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001191 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001192 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1193 : time_controller_(time_controller), test_encoder_(test_encoder) {
1194 RTC_DCHECK(time_controller_);
1195 }
perkj26091b12016-09-01 01:17:40 -07001196
perkj26091b12016-09-01 01:17:40 -07001197 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001198 EXPECT_TRUE(
1199 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1200 }
1201
1202 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1203 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001204 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001205 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001206 return false;
perkj26091b12016-09-01 01:17:40 -07001207 {
Markus Handella3765182020-07-08 13:13:32 +02001208 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001209 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001210 }
1211 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001212 return true;
perkj26091b12016-09-01 01:17:40 -07001213 }
1214
sprangb1ca0732017-02-01 08:38:12 -08001215 void WaitForEncodedFrame(uint32_t expected_width,
1216 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001217 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001218 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001219 }
1220
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001221 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001222 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001223 uint32_t width = 0;
1224 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001225 {
Markus Handella3765182020-07-08 13:13:32 +02001226 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001227 width = last_width_;
1228 height = last_height_;
1229 }
1230 EXPECT_EQ(expected_height, height);
1231 EXPECT_EQ(expected_width, width);
1232 }
1233
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001234 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1235 VideoRotation rotation;
1236 {
Markus Handella3765182020-07-08 13:13:32 +02001237 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001238 rotation = last_rotation_;
1239 }
1240 EXPECT_EQ(expected_rotation, rotation);
1241 }
1242
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001243 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001244
sprangc5d62e22017-04-02 23:53:04 -07001245 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001246 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1247 bool ret = encoded_frame_event_.Wait(timeout_ms);
1248 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1249 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001250 }
1251
perkj26091b12016-09-01 01:17:40 -07001252 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001253 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001254 expect_frames_ = false;
1255 }
1256
asaperssonfab67072017-04-04 05:51:49 -07001257 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001258 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001259 return number_of_reconfigurations_;
1260 }
1261
asaperssonfab67072017-04-04 05:51:49 -07001262 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001263 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001264 return min_transmit_bitrate_bps_;
1265 }
1266
Erik Språngd7329ca2019-02-21 21:19:53 +01001267 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001269 num_expected_layers_ = num_layers;
1270 }
1271
Erik Språngb7cb7b52019-02-26 15:52:33 +01001272 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001273 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001274 return last_capture_time_ms_;
1275 }
1276
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001277 const EncodedImage& GetLastEncodedImage() {
1278 MutexLock lock(&mutex_);
1279 return last_encoded_image_;
1280 }
1281
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001282 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001283 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001284 return std::move(last_encoded_image_data_);
1285 }
1286
Per Kjellanderdcef6412020-10-07 15:09:05 +02001287 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1288 MutexLock lock(&mutex_);
1289 return last_bitrate_allocation_;
1290 }
1291
1292 int number_of_bitrate_allocations() const {
1293 MutexLock lock(&mutex_);
1294 return number_of_bitrate_allocations_;
1295 }
1296
Per Kjellandera9434842020-10-15 17:53:22 +02001297 VideoLayersAllocation GetLastVideoLayersAllocation() {
1298 MutexLock lock(&mutex_);
1299 return last_layers_allocation_;
1300 }
1301
1302 int number_of_layers_allocations() const {
1303 MutexLock lock(&mutex_);
1304 return number_of_layers_allocations_;
1305 }
1306
perkj26091b12016-09-01 01:17:40 -07001307 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001308 Result OnEncodedImage(
1309 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001310 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001311 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001312 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001313 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001314 last_encoded_image_data_ = std::vector<uint8_t>(
1315 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001316 uint32_t timestamp = encoded_image.Timestamp();
1317 if (last_timestamp_ != timestamp) {
1318 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001319 last_width_ = encoded_image._encodedWidth;
1320 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001321 } else {
1322 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001323 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1324 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001325 }
1326 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001327 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001328 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001329 if (num_received_layers_ == num_expected_layers_) {
1330 encoded_frame_event_.Set();
1331 }
sprangb1ca0732017-02-01 08:38:12 -08001332 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001333 }
1334
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001335 void OnEncoderConfigurationChanged(
1336 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001337 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001338 VideoEncoderConfig::ContentType content_type,
1339 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001340 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001341 ++number_of_reconfigurations_;
1342 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1343 }
1344
Per Kjellanderdcef6412020-10-07 15:09:05 +02001345 void OnBitrateAllocationUpdated(
1346 const VideoBitrateAllocation& allocation) override {
1347 MutexLock lock(&mutex_);
1348 ++number_of_bitrate_allocations_;
1349 last_bitrate_allocation_ = allocation;
1350 }
1351
Per Kjellandera9434842020-10-15 17:53:22 +02001352 void OnVideoLayersAllocationUpdated(
1353 VideoLayersAllocation allocation) override {
1354 MutexLock lock(&mutex_);
1355 ++number_of_layers_allocations_;
1356 last_layers_allocation_ = allocation;
1357 rtc::StringBuilder log;
1358 for (const auto& layer : allocation.active_spatial_layers) {
1359 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1360 << "[";
1361 for (const auto target_bitrate :
1362 layer.target_bitrate_per_temporal_layer) {
1363 log << target_bitrate.kbps() << ",";
1364 }
1365 log << "]";
1366 }
1367 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1368 }
1369
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001370 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001371 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001372 TestEncoder* test_encoder_;
1373 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001374 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001375 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001376 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001377 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001378 uint32_t last_height_ = 0;
1379 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001380 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001381 size_t num_expected_layers_ = 1;
1382 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001383 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001384 int number_of_reconfigurations_ = 0;
1385 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001386 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1387 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001388 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1389 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001390 };
1391
Sergey Silkin5ee69672019-07-02 14:18:34 +02001392 class VideoBitrateAllocatorProxyFactory
1393 : public VideoBitrateAllocatorFactory {
1394 public:
1395 VideoBitrateAllocatorProxyFactory()
1396 : bitrate_allocator_factory_(
1397 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1398
1399 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1400 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001402 codec_config_ = codec;
1403 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1404 }
1405
1406 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001407 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001408 return codec_config_;
1409 }
1410
1411 private:
1412 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1413
Markus Handella3765182020-07-08 13:13:32 +02001414 mutable Mutex mutex_;
1415 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001416 };
1417
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001418 Clock* clock() { return time_controller_.GetClock(); }
1419 void AdvanceTime(TimeDelta duration) {
1420 time_controller_.AdvanceTime(duration);
1421 }
1422
1423 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1424
1425 protected:
1426 virtual TaskQueueFactory* GetTaskQueueFactory() {
1427 return time_controller_.GetTaskQueueFactory();
1428 }
1429
1430 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001431 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001432 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001433 int codec_width_;
1434 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001435 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001436 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001437 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001438 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001439 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001440 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001441 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001442 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001443};
1444
mflodmancc3d4422017-08-03 08:27:51 -07001445TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001446 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001447 DataRate::BitsPerSec(kTargetBitrateBps),
1448 DataRate::BitsPerSec(kTargetBitrateBps),
1449 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001450 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001451 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001452 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001453 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001454 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001455}
1456
mflodmancc3d4422017-08-03 08:27:51 -07001457TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001458 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001459 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001460 // The encoder will cache up to one frame for a short duration. Adding two
1461 // frames means that the first frame will be dropped and the second frame will
1462 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001463 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001464 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001465 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001466 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001467
Henrik Boström381d1092020-05-12 18:49:07 +02001468 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001469 DataRate::BitsPerSec(kTargetBitrateBps),
1470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001472
Sebastian Janssona3177052018-04-10 13:05:49 +02001473 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001474 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001475 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1476
1477 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001478 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001479}
1480
mflodmancc3d4422017-08-03 08:27:51 -07001481TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001483 DataRate::BitsPerSec(kTargetBitrateBps),
1484 DataRate::BitsPerSec(kTargetBitrateBps),
1485 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001486 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001487 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001488
Henrik Boström381d1092020-05-12 18:49:07 +02001489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1490 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1491 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001492 // The encoder will cache up to one frame for a short duration. Adding two
1493 // frames means that the first frame will be dropped and the second frame will
1494 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001495 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001496 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001497
Henrik Boström381d1092020-05-12 18:49:07 +02001498 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001499 DataRate::BitsPerSec(kTargetBitrateBps),
1500 DataRate::BitsPerSec(kTargetBitrateBps),
1501 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001502 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001503 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1504 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001505 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001506}
1507
mflodmancc3d4422017-08-03 08:27:51 -07001508TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001510 DataRate::BitsPerSec(kTargetBitrateBps),
1511 DataRate::BitsPerSec(kTargetBitrateBps),
1512 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001513 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001514 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001515
1516 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001518
perkja49cbd32016-09-16 07:53:41 -07001519 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001520 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001521 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001522}
1523
mflodmancc3d4422017-08-03 08:27:51 -07001524TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001526 DataRate::BitsPerSec(kTargetBitrateBps),
1527 DataRate::BitsPerSec(kTargetBitrateBps),
1528 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001529
perkja49cbd32016-09-16 07:53:41 -07001530 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001531 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001532
mflodmancc3d4422017-08-03 08:27:51 -07001533 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001534 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001535 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001536 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1537 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001538}
1539
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001540class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1541 public:
1542 VideoStreamEncoderBlockedTest() {}
1543
1544 TaskQueueFactory* GetTaskQueueFactory() override {
1545 return task_queue_factory_.get();
1546 }
1547
1548 private:
1549 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1550 CreateDefaultTaskQueueFactory();
1551};
1552
1553TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001555 DataRate::BitsPerSec(kTargetBitrateBps),
1556 DataRate::BitsPerSec(kTargetBitrateBps),
1557 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001558
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001559 int dropped_count = 0;
1560 stats_proxy_->SetDroppedFrameCallback(
1561 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1562 ++dropped_count;
1563 });
1564
perkj26091b12016-09-01 01:17:40 -07001565 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001566 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001567 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001568 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1569 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001570 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1571 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001572 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001573 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001574
mflodmancc3d4422017-08-03 08:27:51 -07001575 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001576
1577 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001578}
1579
Henrik Boström56db9ff2021-03-24 09:06:45 +01001580TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001582 DataRate::BitsPerSec(kTargetBitrateBps),
1583 DataRate::BitsPerSec(kTargetBitrateBps),
1584 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001585
1586 rtc::Event frame_destroyed_event;
1587 video_source_.IncomingCapturedFrame(
1588 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001589 WaitForEncodedFrame(1);
1590 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1591 fake_encoder_.GetLastInputPixelFormat());
1592 EXPECT_EQ(fake_encoder_.codec_config().width,
1593 fake_encoder_.GetLastInputWidth());
1594 EXPECT_EQ(fake_encoder_.codec_config().height,
1595 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001596 video_stream_encoder_->Stop();
1597}
1598
Henrik Boström56db9ff2021-03-24 09:06:45 +01001599TEST_F(VideoStreamEncoderTest,
1600 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001601 // Use the cropping factory.
1602 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001603 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001604 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1605 kMaxPayloadLength);
1606 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1607
1608 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001610 DataRate::BitsPerSec(kTargetBitrateBps),
1611 DataRate::BitsPerSec(kTargetBitrateBps),
1612 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001613 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1614 WaitForEncodedFrame(1);
1615 // The encoder will have been configured once.
1616 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1617 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1618 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1619
1620 // Now send in a fake frame that needs to be cropped as the width/height
1621 // aren't divisible by 4 (see CreateEncoderStreams above).
1622 rtc::Event frame_destroyed_event;
1623 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1624 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001625 WaitForEncodedFrame(2);
1626 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1627 fake_encoder_.GetLastInputPixelFormat());
1628 EXPECT_EQ(fake_encoder_.codec_config().width,
1629 fake_encoder_.GetLastInputWidth());
1630 EXPECT_EQ(fake_encoder_.codec_config().height,
1631 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001632 video_stream_encoder_->Stop();
1633}
1634
Evan Shrubsole895556e2020-10-05 09:15:13 +02001635TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1637 DataRate::BitsPerSec(kTargetBitrateBps),
1638 DataRate::BitsPerSec(kTargetBitrateBps),
1639 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1640
1641 video_source_.IncomingCapturedFrame(
1642 CreateNV12Frame(1, codec_width_, codec_height_));
1643 WaitForEncodedFrame(1);
1644 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1645 fake_encoder_.GetLastInputPixelFormat());
1646 video_stream_encoder_->Stop();
1647}
1648
Henrik Boström56db9ff2021-03-24 09:06:45 +01001649TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1651 DataRate::BitsPerSec(kTargetBitrateBps),
1652 DataRate::BitsPerSec(kTargetBitrateBps),
1653 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1654
1655 fake_encoder_.SetPreferredPixelFormats({});
1656
1657 rtc::Event frame_destroyed_event;
1658 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1659 1, &frame_destroyed_event, codec_width_, codec_height_));
1660 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001661 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001662 fake_encoder_.GetLastInputPixelFormat());
1663 video_stream_encoder_->Stop();
1664}
1665
Henrik Boström56db9ff2021-03-24 09:06:45 +01001666TEST_F(VideoStreamEncoderTest,
1667 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001668 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1669 DataRate::BitsPerSec(kTargetBitrateBps),
1670 DataRate::BitsPerSec(kTargetBitrateBps),
1671 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1672
1673 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1674
1675 rtc::Event frame_destroyed_event;
1676 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1677 1, &frame_destroyed_event, codec_width_, codec_height_));
1678 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001679 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001680 fake_encoder_.GetLastInputPixelFormat());
1681 video_stream_encoder_->Stop();
1682}
1683
Henrik Boström56db9ff2021-03-24 09:06:45 +01001684TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1686 DataRate::BitsPerSec(kTargetBitrateBps),
1687 DataRate::BitsPerSec(kTargetBitrateBps),
1688 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1689
1690 // Fake NV12 native frame does not allow mapping to I444.
1691 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1692
1693 rtc::Event frame_destroyed_event;
1694 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1695 1, &frame_destroyed_event, codec_width_, codec_height_));
1696 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001697 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001698 fake_encoder_.GetLastInputPixelFormat());
1699 video_stream_encoder_->Stop();
1700}
1701
Henrik Boström56db9ff2021-03-24 09:06:45 +01001702TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1704 DataRate::BitsPerSec(kTargetBitrateBps),
1705 DataRate::BitsPerSec(kTargetBitrateBps),
1706 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1707
1708 rtc::Event frame_destroyed_event;
1709 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1710 1, &frame_destroyed_event, codec_width_, codec_height_));
1711 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001712 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001713 fake_encoder_.GetLastInputPixelFormat());
1714 video_stream_encoder_->Stop();
1715}
1716
Ying Wang9b881ab2020-02-07 14:29:32 +01001717TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001719 DataRate::BitsPerSec(kTargetBitrateBps),
1720 DataRate::BitsPerSec(kTargetBitrateBps),
1721 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001722 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1723 WaitForEncodedFrame(1);
1724
Henrik Boström381d1092020-05-12 18:49:07 +02001725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001726 DataRate::BitsPerSec(kTargetBitrateBps),
1727 DataRate::BitsPerSec(kTargetBitrateBps),
1728 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001729 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1730 // frames. Adding two frames means that the first frame will be dropped and
1731 // the second frame will be sent to the encoder.
1732 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1733 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1734 WaitForEncodedFrame(3);
1735 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1736 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1737 WaitForEncodedFrame(5);
1738 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1739 video_stream_encoder_->Stop();
1740}
1741
mflodmancc3d4422017-08-03 08:27:51 -07001742TEST_F(VideoStreamEncoderTest,
1743 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001745 DataRate::BitsPerSec(kTargetBitrateBps),
1746 DataRate::BitsPerSec(kTargetBitrateBps),
1747 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001748 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001749
1750 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001751 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001752 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001753 // The encoder will have been configured once when the first frame is
1754 // received.
1755 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001756
1757 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001758 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001759 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001760 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001761 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001762
1763 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001764 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001765 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001766 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001767 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001768
mflodmancc3d4422017-08-03 08:27:51 -07001769 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001770}
1771
mflodmancc3d4422017-08-03 08:27:51 -07001772TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001774 DataRate::BitsPerSec(kTargetBitrateBps),
1775 DataRate::BitsPerSec(kTargetBitrateBps),
1776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001777
1778 // Capture a frame and wait for it to synchronize with the encoder thread.
1779 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001780 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001781 // The encoder will have been configured once.
1782 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001783 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1784 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1785
1786 codec_width_ *= 2;
1787 codec_height_ *= 2;
1788 // Capture a frame with a higher resolution and wait for it to synchronize
1789 // with the encoder thread.
1790 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001791 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001792 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1793 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001794 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001795
mflodmancc3d4422017-08-03 08:27:51 -07001796 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001797}
1798
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001799TEST_F(VideoStreamEncoderTest,
1800 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001802 DataRate::BitsPerSec(kTargetBitrateBps),
1803 DataRate::BitsPerSec(kTargetBitrateBps),
1804 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001805
1806 // Capture a frame and wait for it to synchronize with the encoder thread.
1807 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1808 WaitForEncodedFrame(1);
1809
1810 VideoEncoderConfig video_encoder_config;
1811 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1812 // Changing the max payload data length recreates encoder.
1813 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1814 kMaxPayloadLength / 2);
1815
1816 // Capture a frame and wait for it to synchronize with the encoder thread.
1817 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1818 WaitForEncodedFrame(2);
1819 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1820
1821 video_stream_encoder_->Stop();
1822}
1823
Sergey Silkin5ee69672019-07-02 14:18:34 +02001824TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001826 DataRate::BitsPerSec(kTargetBitrateBps),
1827 DataRate::BitsPerSec(kTargetBitrateBps),
1828 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001829
1830 VideoEncoderConfig video_encoder_config;
1831 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1832 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1833 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1834 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1835 kMaxPayloadLength);
1836
1837 // Capture a frame and wait for it to synchronize with the encoder thread.
1838 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1839 WaitForEncodedFrame(1);
1840 // The encoder will have been configured once when the first frame is
1841 // received.
1842 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1843 EXPECT_EQ(kTargetBitrateBps,
1844 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1845 EXPECT_EQ(kStartBitrateBps,
1846 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1847
Sergey Silkin6456e352019-07-08 17:56:40 +02001848 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1849 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001850 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1851 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1852 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1853 kMaxPayloadLength);
1854
1855 // Capture a frame and wait for it to synchronize with the encoder thread.
1856 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1857 WaitForEncodedFrame(2);
1858 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1859 // Bitrate limits have changed - rate allocator should be reconfigured,
1860 // encoder should not be reconfigured.
1861 EXPECT_EQ(kTargetBitrateBps * 2,
1862 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1863 EXPECT_EQ(kStartBitrateBps * 2,
1864 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1865 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1866
1867 video_stream_encoder_->Stop();
1868}
1869
Sergey Silkin6456e352019-07-08 17:56:40 +02001870TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001871 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001873 DataRate::BitsPerSec(kTargetBitrateBps),
1874 DataRate::BitsPerSec(kTargetBitrateBps),
1875 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001876
Sergey Silkincd02eba2020-01-20 14:48:40 +01001877 const uint32_t kMinEncBitrateKbps = 100;
1878 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001879 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001880 /*frame_size_pixels=*/codec_width_ * codec_height_,
1881 /*min_start_bitrate_bps=*/0,
1882 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1883 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001884 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1885
Sergey Silkincd02eba2020-01-20 14:48:40 +01001886 VideoEncoderConfig video_encoder_config;
1887 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1888 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1889 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1890 (kMinEncBitrateKbps + 1) * 1000;
1891 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1892 kMaxPayloadLength);
1893
1894 // When both encoder and app provide bitrate limits, the intersection of
1895 // provided sets should be used.
1896 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1897 WaitForEncodedFrame(1);
1898 EXPECT_EQ(kMaxEncBitrateKbps,
1899 bitrate_allocator_factory_.codec_config().maxBitrate);
1900 EXPECT_EQ(kMinEncBitrateKbps + 1,
1901 bitrate_allocator_factory_.codec_config().minBitrate);
1902
1903 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1904 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1905 (kMinEncBitrateKbps - 1) * 1000;
1906 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1907 kMaxPayloadLength);
1908 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001909 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001910 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001911 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001912 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001913 bitrate_allocator_factory_.codec_config().minBitrate);
1914
Sergey Silkincd02eba2020-01-20 14:48:40 +01001915 video_stream_encoder_->Stop();
1916}
1917
1918TEST_F(VideoStreamEncoderTest,
1919 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001921 DataRate::BitsPerSec(kTargetBitrateBps),
1922 DataRate::BitsPerSec(kTargetBitrateBps),
1923 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001924
1925 const uint32_t kMinAppBitrateKbps = 100;
1926 const uint32_t kMaxAppBitrateKbps = 200;
1927 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1928 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1929 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1930 /*frame_size_pixels=*/codec_width_ * codec_height_,
1931 /*min_start_bitrate_bps=*/0,
1932 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1933 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1934 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1935
1936 VideoEncoderConfig video_encoder_config;
1937 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1938 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1939 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1940 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001941 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1942 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001943
Sergey Silkincd02eba2020-01-20 14:48:40 +01001944 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1945 WaitForEncodedFrame(1);
1946 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001947 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001948 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001950
1951 video_stream_encoder_->Stop();
1952}
1953
1954TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001955 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001957 DataRate::BitsPerSec(kTargetBitrateBps),
1958 DataRate::BitsPerSec(kTargetBitrateBps),
1959 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001960
1961 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001963 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001965 fake_encoder_.SetResolutionBitrateLimits(
1966 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1967
1968 VideoEncoderConfig video_encoder_config;
1969 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1970 video_encoder_config.max_bitrate_bps = 0;
1971 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1972 kMaxPayloadLength);
1973
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001974 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001975 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1976 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001977 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1978 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001979 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1980 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1981
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001982 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001983 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1984 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001985 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1986 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001987 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1988 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1989
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001991 // encoder for 360p should be used.
1992 video_source_.IncomingCapturedFrame(
1993 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1994 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001997 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1998 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1999
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002000 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002001 // ignored.
2002 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2003 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002006 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2007 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2009 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002010 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2011 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2012
2013 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2014 // for 270p should be used.
2015 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2016 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2018 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002019 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2020 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2021
2022 video_stream_encoder_->Stop();
2023}
2024
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002027 DataRate::BitsPerSec(kTargetBitrateBps),
2028 DataRate::BitsPerSec(kTargetBitrateBps),
2029 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002030
2031 VideoEncoderConfig video_encoder_config;
2032 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2033 video_encoder_config.max_bitrate_bps = 0;
2034 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2035 kMaxPayloadLength);
2036
2037 // Encode 720p frame to get the default encoder target bitrate.
2038 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2039 WaitForEncodedFrame(1);
2040 const uint32_t kDefaultTargetBitrateFor720pKbps =
2041 bitrate_allocator_factory_.codec_config()
2042 .simulcastStream[0]
2043 .targetBitrate;
2044
2045 // Set the max recommended encoder bitrate to something lower than the default
2046 // target bitrate.
2047 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2048 1280 * 720, 10 * 1000, 10 * 1000,
2049 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2050 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2051
2052 // Change resolution to trigger encoder reinitialization.
2053 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2054 WaitForEncodedFrame(2);
2055 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2056 WaitForEncodedFrame(3);
2057
2058 // Ensure the target bitrate is capped by the max bitrate.
2059 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2060 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2061 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2062 .simulcastStream[0]
2063 .targetBitrate *
2064 1000,
2065 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2066
2067 video_stream_encoder_->Stop();
2068}
2069
Åsa Perssona7e34d32021-01-20 15:36:13 +01002070TEST_F(VideoStreamEncoderTest,
2071 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2072 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2073 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2074 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2075 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2076 fake_encoder_.SetResolutionBitrateLimits(
2077 {kEncoderLimits270p, kEncoderLimits360p});
2078
2079 // Two streams, highest stream active.
2080 VideoEncoderConfig config;
2081 const int kNumStreams = 2;
2082 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2083 config.max_bitrate_bps = 0;
2084 config.simulcast_layers[0].active = false;
2085 config.simulcast_layers[1].active = true;
2086 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002087 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002088 "VP8", /*max qp*/ 56, /*screencast*/ false,
2089 /*screenshare enabled*/ false);
2090 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2091
2092 // The encoder bitrate limits for 270p should be used.
2093 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2094 EXPECT_FALSE(WaitForFrame(1000));
2095 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2096 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2097 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2098 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2099 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2100
2101 // The encoder bitrate limits for 360p should be used.
2102 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2103 EXPECT_FALSE(WaitForFrame(1000));
2104 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2105 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2106 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2107 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2108
2109 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2110 video_source_.IncomingCapturedFrame(
2111 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2112 EXPECT_FALSE(WaitForFrame(1000));
2113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2114 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2115 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2116 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2117
2118 // Resolution higher than 360p. Encoder limits should be ignored.
2119 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2120 EXPECT_FALSE(WaitForFrame(1000));
2121 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2122 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2123 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2124 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2125 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2126 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2127 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2128 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2129
2130 // Resolution lower than 270p. The encoder limits for 270p should be used.
2131 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2132 EXPECT_FALSE(WaitForFrame(1000));
2133 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2134 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2135 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2136 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2137
2138 video_stream_encoder_->Stop();
2139}
2140
2141TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002142 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2143 // Two streams, highest stream active.
2144 VideoEncoderConfig config;
2145 const int kNumStreams = 2;
2146 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2147 config.max_bitrate_bps = 0;
2148 config.simulcast_layers[0].active = false;
2149 config.simulcast_layers[1].active = true;
2150 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002151 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002152 "VP8", /*max qp*/ 56, /*screencast*/ false,
2153 /*screenshare enabled*/ false);
2154 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2155
2156 // Default bitrate limits for 270p should be used.
2157 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2158 kDefaultLimits270p =
2159 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002160 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002161 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2162 EXPECT_FALSE(WaitForFrame(1000));
2163 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2164 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2165 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2166 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2167 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2168
2169 // Default bitrate limits for 360p should be used.
2170 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2171 kDefaultLimits360p =
2172 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002173 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002174 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2177 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2178 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2179 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2180
2181 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2182 video_source_.IncomingCapturedFrame(
2183 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2184 EXPECT_FALSE(WaitForFrame(1000));
2185 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2186 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2187 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2188 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2189
2190 // Default bitrate limits for 540p should be used.
2191 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2192 kDefaultLimits540p =
2193 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002194 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002195 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2196 EXPECT_FALSE(WaitForFrame(1000));
2197 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2198 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2199 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2200 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2201
2202 video_stream_encoder_->Stop();
2203}
2204
2205TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2207 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2208 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2209 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2210 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2211 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2212 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2213 fake_encoder_.SetResolutionBitrateLimits(
2214 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2215
2216 // Three streams, middle stream active.
2217 VideoEncoderConfig config;
2218 const int kNumStreams = 3;
2219 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2220 config.simulcast_layers[0].active = false;
2221 config.simulcast_layers[1].active = true;
2222 config.simulcast_layers[2].active = false;
2223 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002224 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225 "VP8", /*max qp*/ 56, /*screencast*/ false,
2226 /*screenshare enabled*/ false);
2227 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2228
2229 // The encoder bitrate limits for 360p should be used.
2230 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2231 EXPECT_FALSE(WaitForFrame(1000));
2232 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2233 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2234 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2235 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2236 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2237
2238 // The encoder bitrate limits for 270p should be used.
2239 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2240 EXPECT_FALSE(WaitForFrame(1000));
2241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2242 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2244 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2245
2246 video_stream_encoder_->Stop();
2247}
2248
2249TEST_F(VideoStreamEncoderTest,
2250 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2251 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2252 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2253 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2254 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2255 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2256 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2257 fake_encoder_.SetResolutionBitrateLimits(
2258 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2259
2260 // Three streams, lowest stream active.
2261 VideoEncoderConfig config;
2262 const int kNumStreams = 3;
2263 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2264 config.simulcast_layers[0].active = true;
2265 config.simulcast_layers[1].active = false;
2266 config.simulcast_layers[2].active = false;
2267 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002268 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002269 "VP8", /*max qp*/ 56, /*screencast*/ false,
2270 /*screenshare enabled*/ false);
2271 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2272
2273 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2274 // on lowest stream, limits for 270p should not be used
2275 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2276 EXPECT_FALSE(WaitForFrame(1000));
2277 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2278 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2279 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2280 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2281 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2282
2283 video_stream_encoder_->Stop();
2284}
2285
2286TEST_F(VideoStreamEncoderTest,
2287 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2288 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2289 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2290 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2291 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2292 fake_encoder_.SetResolutionBitrateLimits(
2293 {kEncoderLimits270p, kEncoderLimits360p});
2294 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2295
2296 // Two streams, highest stream active.
2297 VideoEncoderConfig config;
2298 const int kNumStreams = 2;
2299 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2300 config.simulcast_layers[0].active = false;
2301 config.simulcast_layers[1].active = true;
2302 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2303 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002304 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002305 "VP8", /*max qp*/ 56, /*screencast*/ false,
2306 /*screenshare enabled*/ false);
2307 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2308
2309 // The encoder bitrate limits for 270p should be used.
2310 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2311 EXPECT_FALSE(WaitForFrame(1000));
2312 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2314 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2315 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2316 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2317
2318 // The max configured bitrate is less than the encoder limit for 360p.
2319 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2320 EXPECT_FALSE(WaitForFrame(1000));
2321 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2322 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2323 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2324 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2325
2326 video_stream_encoder_->Stop();
2327}
2328
mflodmancc3d4422017-08-03 08:27:51 -07002329TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002330 EXPECT_TRUE(video_source_.has_sinks());
2331 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002332 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002333 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002334 EXPECT_FALSE(video_source_.has_sinks());
2335 EXPECT_TRUE(new_video_source.has_sinks());
2336
mflodmancc3d4422017-08-03 08:27:51 -07002337 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002338}
2339
mflodmancc3d4422017-08-03 08:27:51 -07002340TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002341 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002342 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002343 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002344 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002345}
2346
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002347class ResolutionAlignmentTest
2348 : public VideoStreamEncoderTest,
2349 public ::testing::WithParamInterface<
2350 ::testing::tuple<int, std::vector<double>>> {
2351 public:
2352 ResolutionAlignmentTest()
2353 : requested_alignment_(::testing::get<0>(GetParam())),
2354 scale_factors_(::testing::get<1>(GetParam())) {}
2355
2356 protected:
2357 const int requested_alignment_;
2358 const std::vector<double> scale_factors_;
2359};
2360
2361INSTANTIATE_TEST_SUITE_P(
2362 AlignmentAndScaleFactors,
2363 ResolutionAlignmentTest,
2364 ::testing::Combine(
2365 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2366 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2367 std::vector<double>{-1.0, -1.0},
2368 std::vector<double>{-1.0, -1.0, -1.0},
2369 std::vector<double>{4.0, 2.0, 1.0},
2370 std::vector<double>{9999.0, -1.0, 1.0},
2371 std::vector<double>{3.99, 2.01, 1.0},
2372 std::vector<double>{4.9, 1.7, 1.25},
2373 std::vector<double>{10.0, 4.0, 3.0},
2374 std::vector<double>{1.75, 3.5},
2375 std::vector<double>{1.5, 2.5},
2376 std::vector<double>{1.3, 1.0})));
2377
2378TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2379 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002380 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002381 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2382 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2383
2384 // Fill config with the scaling factor by which to reduce encoding size.
2385 const int num_streams = scale_factors_.size();
2386 VideoEncoderConfig config;
2387 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2388 for (int i = 0; i < num_streams; ++i) {
2389 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2390 }
2391 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002392 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002393 "VP8", /*max qp*/ 56, /*screencast*/ false,
2394 /*screenshare enabled*/ false);
2395 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2396
Henrik Boström381d1092020-05-12 18:49:07 +02002397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002398 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2399 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2400 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2401 // Wait for all layers before triggering event.
2402 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002403
2404 // On the 1st frame, we should have initialized the encoder and
2405 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002406 int64_t timestamp_ms = kFrameIntervalMs;
2407 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2408 WaitForEncodedFrame(timestamp_ms);
2409 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002410
2411 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2412 // (It's up the to the encoder to potentially drop the previous frame,
2413 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002414 timestamp_ms += kFrameIntervalMs;
2415 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2416 WaitForEncodedFrame(timestamp_ms);
2417 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2418
2419 VideoCodec codec = fake_encoder_.video_codec();
2420 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2421 // Frame size should be a multiple of the requested alignment.
2422 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2423 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2424 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2425 // Aspect ratio should match.
2426 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2427 codec.height * codec.simulcastStream[i].width);
2428 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002429
2430 video_stream_encoder_->Stop();
2431}
2432
Jonathan Yubc771b72017-12-08 17:04:29 -08002433TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2434 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002435 const int kWidth = 1280;
2436 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002437
2438 // We rely on the automatic resolution adaptation, but we handle framerate
2439 // adaptation manually by mocking the stats proxy.
2440 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002441
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002442 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002444 DataRate::BitsPerSec(kTargetBitrateBps),
2445 DataRate::BitsPerSec(kTargetBitrateBps),
2446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002447 video_stream_encoder_->SetSource(&video_source_,
2448 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002449 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002450 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002451 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002452 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2453
Jonathan Yubc771b72017-12-08 17:04:29 -08002454 // Adapt down as far as possible.
2455 rtc::VideoSinkWants last_wants;
2456 int64_t t = 1;
2457 int loop_count = 0;
2458 do {
2459 ++loop_count;
2460 last_wants = video_source_.sink_wants();
2461
2462 // Simulate the framerate we've been asked to adapt to.
2463 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2464 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2465 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2466 mock_stats.input_frame_rate = fps;
2467 stats_proxy_->SetMockStats(mock_stats);
2468
2469 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2470 sink_.WaitForEncodedFrame(t);
2471 t += frame_interval_ms;
2472
mflodmancc3d4422017-08-03 08:27:51 -07002473 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002474 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002475 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002476 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2477 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002478 } while (video_source_.sink_wants().max_pixel_count <
2479 last_wants.max_pixel_count ||
2480 video_source_.sink_wants().max_framerate_fps <
2481 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002482
Jonathan Yubc771b72017-12-08 17:04:29 -08002483 // Verify that we've adapted all the way down.
2484 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002485 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002486 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2487 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002488 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002489 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2490 *video_source_.last_sent_height());
2491 EXPECT_EQ(kMinBalancedFramerateFps,
2492 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002493
Jonathan Yubc771b72017-12-08 17:04:29 -08002494 // Adapt back up the same number of times we adapted down.
2495 for (int i = 0; i < loop_count - 1; ++i) {
2496 last_wants = video_source_.sink_wants();
2497
2498 // Simulate the framerate we've been asked to adapt to.
2499 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2500 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2501 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2502 mock_stats.input_frame_rate = fps;
2503 stats_proxy_->SetMockStats(mock_stats);
2504
2505 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2506 sink_.WaitForEncodedFrame(t);
2507 t += frame_interval_ms;
2508
Henrik Boström91aa7322020-04-28 12:24:33 +02002509 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002510 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002511 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002512 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2513 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002514 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2515 last_wants.max_pixel_count ||
2516 video_source_.sink_wants().max_framerate_fps >
2517 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002518 }
2519
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002520 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002521 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002522 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002523 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2524 EXPECT_EQ((loop_count - 1) * 2,
2525 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002526
mflodmancc3d4422017-08-03 08:27:51 -07002527 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002528}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002529
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002530TEST_F(VideoStreamEncoderTest,
2531 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2532 video_stream_encoder_->OnBitrateUpdated(
2533 DataRate::BitsPerSec(kTargetBitrateBps),
2534 DataRate::BitsPerSec(kTargetBitrateBps),
2535 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002536 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002537
2538 const int kFrameWidth = 1280;
2539 const int kFrameHeight = 720;
2540
2541 int64_t ntp_time = kFrameIntervalMs;
2542
2543 // Force an input frame rate to be available, or the adaptation call won't
2544 // know what framerate to adapt form.
2545 const int kInputFps = 30;
2546 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2547 stats.input_frame_rate = kInputFps;
2548 stats_proxy_->SetMockStats(stats);
2549
2550 video_source_.set_adaptation_enabled(true);
2551 video_stream_encoder_->SetSource(
2552 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002553 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 // Trigger CPU overuse.
2560 video_stream_encoder_->TriggerCpuOveruse();
2561 video_source_.IncomingCapturedFrame(
2562 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2563 sink_.WaitForEncodedFrame(ntp_time);
2564 ntp_time += kFrameIntervalMs;
2565
2566 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2567 EXPECT_EQ(std::numeric_limits<int>::max(),
2568 video_source_.sink_wants().max_pixel_count);
2569 // Some framerate constraint should be set.
2570 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2571 EXPECT_LT(restricted_fps, kInputFps);
2572 video_source_.IncomingCapturedFrame(
2573 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2574 sink_.WaitForEncodedFrame(ntp_time);
2575 ntp_time += 100;
2576
Henrik Boström2671dac2020-05-19 16:29:09 +02002577 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002578 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2579 // Give the encoder queue time to process the change in degradation preference
2580 // by waiting for an encoded frame.
2581 video_source_.IncomingCapturedFrame(
2582 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2583 sink_.WaitForEncodedFrame(ntp_time);
2584 ntp_time += kFrameIntervalMs;
2585
2586 video_stream_encoder_->TriggerQualityLow();
2587 video_source_.IncomingCapturedFrame(
2588 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2589 sink_.WaitForEncodedFrame(ntp_time);
2590 ntp_time += kFrameIntervalMs;
2591
2592 // Some resolution constraint should be set.
2593 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2594 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2595 kFrameWidth * kFrameHeight);
2596 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2597
2598 int pixel_count = video_source_.sink_wants().max_pixel_count;
2599 // Triggering a CPU underuse should not change the sink wants since it has
2600 // not been overused for resolution since we changed degradation preference.
2601 video_stream_encoder_->TriggerCpuUnderuse();
2602 video_source_.IncomingCapturedFrame(
2603 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2604 sink_.WaitForEncodedFrame(ntp_time);
2605 ntp_time += kFrameIntervalMs;
2606 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2607 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2608
Evan Shrubsole64469032020-06-11 10:45:29 +02002609 // Change the degradation preference back. CPU underuse should not adapt since
2610 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002611 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002612 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += 100;
2617 // Resolution adaptations is gone after changing degradation preference.
2618 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2619 EXPECT_EQ(std::numeric_limits<int>::max(),
2620 video_source_.sink_wants().max_pixel_count);
2621 // The fps adaptation from above is now back.
2622 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2623
2624 // Trigger CPU underuse.
2625 video_stream_encoder_->TriggerCpuUnderuse();
2626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002630 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2631
2632 // Trigger QP underuse, fps should return to normal.
2633 video_stream_encoder_->TriggerQualityHigh();
2634 video_source_.IncomingCapturedFrame(
2635 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2636 sink_.WaitForEncodedFrame(ntp_time);
2637 ntp_time += kFrameIntervalMs;
2638 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002639
2640 video_stream_encoder_->Stop();
2641}
2642
mflodmancc3d4422017-08-03 08:27:51 -07002643TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002645 DataRate::BitsPerSec(kTargetBitrateBps),
2646 DataRate::BitsPerSec(kTargetBitrateBps),
2647 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002648 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002649
sprangc5d62e22017-04-02 23:53:04 -07002650 const int kFrameWidth = 1280;
2651 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002652
Åsa Persson8c1bf952018-09-13 10:42:19 +02002653 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002654
kthelgason5e13d412016-12-01 03:59:51 -08002655 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002656 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002657 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002658 frame_timestamp += kFrameIntervalMs;
2659
perkj803d97f2016-11-01 11:45:46 -07002660 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002661 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002662 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002663 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002664 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002665 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002666
asapersson0944a802017-04-07 00:57:58 -07002667 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002668 // wanted resolution.
2669 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2670 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2671 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002672 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002673
2674 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002675 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002676 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002677 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002678 // Give the encoder queue time to process the change in degradation preference
2679 // by waiting for an encoded frame.
2680 new_video_source.IncomingCapturedFrame(
2681 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2682 sink_.WaitForEncodedFrame(frame_timestamp);
2683 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002684 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002685 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002686
sprangc5d62e22017-04-02 23:53:04 -07002687 // Force an input frame rate to be available, or the adaptation call won't
2688 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002689 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002690 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002691 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002692 stats_proxy_->SetMockStats(stats);
2693
mflodmancc3d4422017-08-03 08:27:51 -07002694 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002695 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002696 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002697 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002698 frame_timestamp += kFrameIntervalMs;
2699
2700 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002701 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002702 EXPECT_EQ(std::numeric_limits<int>::max(),
2703 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002704 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002705
asapersson02465b82017-04-10 01:12:52 -07002706 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002707 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2708 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002709 // Give the encoder queue time to process the change in degradation preference
2710 // by waiting for an encoded frame.
2711 new_video_source.IncomingCapturedFrame(
2712 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2713 sink_.WaitForEncodedFrame(frame_timestamp);
2714 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002715 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002716
mflodmancc3d4422017-08-03 08:27:51 -07002717 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002718 new_video_source.IncomingCapturedFrame(
2719 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002720 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002721 frame_timestamp += kFrameIntervalMs;
2722
2723 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002724 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002725
2726 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002727 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002728 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002729 // Give the encoder queue time to process the change in degradation preference
2730 // by waiting for an encoded frame.
2731 new_video_source.IncomingCapturedFrame(
2732 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2733 sink_.WaitForEncodedFrame(frame_timestamp);
2734 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002735 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2736 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002737 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002738 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002739
2740 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002741 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002742 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002743 // Give the encoder queue time to process the change in degradation preference
2744 // by waiting for an encoded frame.
2745 new_video_source.IncomingCapturedFrame(
2746 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2747 sink_.WaitForEncodedFrame(frame_timestamp);
2748 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002749 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2750 EXPECT_EQ(std::numeric_limits<int>::max(),
2751 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002752 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002753
mflodmancc3d4422017-08-03 08:27:51 -07002754 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002755}
2756
mflodmancc3d4422017-08-03 08:27:51 -07002757TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002759 DataRate::BitsPerSec(kTargetBitrateBps),
2760 DataRate::BitsPerSec(kTargetBitrateBps),
2761 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002762
asaperssonfab67072017-04-04 05:51:49 -07002763 const int kWidth = 1280;
2764 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002765 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002766 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002767 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2768 EXPECT_FALSE(stats.bw_limited_resolution);
2769 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2770
2771 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002772 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002773 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002774 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002775
2776 stats = stats_proxy_->GetStats();
2777 EXPECT_TRUE(stats.bw_limited_resolution);
2778 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2779
2780 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002781 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002782 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002784
2785 stats = stats_proxy_->GetStats();
2786 EXPECT_FALSE(stats.bw_limited_resolution);
2787 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2788 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2789
mflodmancc3d4422017-08-03 08:27:51 -07002790 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002791}
2792
mflodmancc3d4422017-08-03 08:27:51 -07002793TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002795 DataRate::BitsPerSec(kTargetBitrateBps),
2796 DataRate::BitsPerSec(kTargetBitrateBps),
2797 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002798
2799 const int kWidth = 1280;
2800 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002801 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002802 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002803 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2804 EXPECT_FALSE(stats.cpu_limited_resolution);
2805 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2806
2807 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002808 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002809 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002810 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002811
2812 stats = stats_proxy_->GetStats();
2813 EXPECT_TRUE(stats.cpu_limited_resolution);
2814 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2815
2816 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002817 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002818 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002820
2821 stats = stats_proxy_->GetStats();
2822 EXPECT_FALSE(stats.cpu_limited_resolution);
2823 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002824 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002825
mflodmancc3d4422017-08-03 08:27:51 -07002826 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002827}
2828
mflodmancc3d4422017-08-03 08:27:51 -07002829TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002831 DataRate::BitsPerSec(kTargetBitrateBps),
2832 DataRate::BitsPerSec(kTargetBitrateBps),
2833 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002834
asaperssonfab67072017-04-04 05:51:49 -07002835 const int kWidth = 1280;
2836 const int kHeight = 720;
2837 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002838 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002839 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002840 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002841 EXPECT_FALSE(stats.cpu_limited_resolution);
2842 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2843
asaperssonfab67072017-04-04 05:51:49 -07002844 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002845 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002846 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002847 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002848 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002849 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002850 EXPECT_TRUE(stats.cpu_limited_resolution);
2851 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2852
2853 // Set new source with adaptation still enabled.
2854 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002855 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002856 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002857
asaperssonfab67072017-04-04 05:51:49 -07002858 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002859 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002860 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002861 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002862 EXPECT_TRUE(stats.cpu_limited_resolution);
2863 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2864
2865 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002866 video_stream_encoder_->SetSource(&new_video_source,
2867 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002868
asaperssonfab67072017-04-04 05:51:49 -07002869 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002870 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002871 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002872 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002873 EXPECT_FALSE(stats.cpu_limited_resolution);
2874 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2875
2876 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002878 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002879
asaperssonfab67072017-04-04 05:51:49 -07002880 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002881 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002882 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002883 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002884 EXPECT_TRUE(stats.cpu_limited_resolution);
2885 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2886
asaperssonfab67072017-04-04 05:51:49 -07002887 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002888 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002889 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002890 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002891 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002892 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002893 EXPECT_FALSE(stats.cpu_limited_resolution);
2894 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002895 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002896
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002898}
2899
mflodmancc3d4422017-08-03 08:27:51 -07002900TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002902 DataRate::BitsPerSec(kTargetBitrateBps),
2903 DataRate::BitsPerSec(kTargetBitrateBps),
2904 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002905
asaperssonfab67072017-04-04 05:51:49 -07002906 const int kWidth = 1280;
2907 const int kHeight = 720;
2908 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002909 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002910 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002911 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002912 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002913 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002914
2915 // Set new source with adaptation still enabled.
2916 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002917 video_stream_encoder_->SetSource(&new_video_source,
2918 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002919
asaperssonfab67072017-04-04 05:51:49 -07002920 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002922 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002923 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002924 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002925 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002926
asaperssonfab67072017-04-04 05:51:49 -07002927 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002929 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002930 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002931 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002932 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002933 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002934 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002935
asaperssonfab67072017-04-04 05:51:49 -07002936 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002937 video_stream_encoder_->SetSource(&new_video_source,
2938 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002939
asaperssonfab67072017-04-04 05:51:49 -07002940 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002941 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002942 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002943 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002944 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002945 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002946
asapersson02465b82017-04-10 01:12:52 -07002947 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002948 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002949 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002950
asaperssonfab67072017-04-04 05:51:49 -07002951 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002952 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002953 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002954 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002955 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002956 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2957 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002958
mflodmancc3d4422017-08-03 08:27:51 -07002959 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002960}
2961
mflodmancc3d4422017-08-03 08:27:51 -07002962TEST_F(VideoStreamEncoderTest,
2963 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002965 DataRate::BitsPerSec(kTargetBitrateBps),
2966 DataRate::BitsPerSec(kTargetBitrateBps),
2967 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002968
2969 const int kWidth = 1280;
2970 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002971 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002972 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002973 video_source_.IncomingCapturedFrame(
2974 CreateFrame(timestamp_ms, kWidth, kHeight));
2975 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002976 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2978 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2979
2980 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002982 timestamp_ms += kFrameIntervalMs;
2983 video_source_.IncomingCapturedFrame(
2984 CreateFrame(timestamp_ms, kWidth, kHeight));
2985 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002986 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2988 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2989
2990 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002991 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002992 timestamp_ms += kFrameIntervalMs;
2993 video_source_.IncomingCapturedFrame(
2994 CreateFrame(timestamp_ms, kWidth, kHeight));
2995 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002996 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2997 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2998 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2999
Niels Möller4db138e2018-04-19 09:04:13 +02003000 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003001 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003002
3003 VideoEncoderConfig video_encoder_config;
3004 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3005 // Make format different, to force recreation of encoder.
3006 video_encoder_config.video_format.parameters["foo"] = "foo";
3007 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003008 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003009 timestamp_ms += kFrameIntervalMs;
3010 video_source_.IncomingCapturedFrame(
3011 CreateFrame(timestamp_ms, kWidth, kHeight));
3012 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003013 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3014 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3015 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3016
mflodmancc3d4422017-08-03 08:27:51 -07003017 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003018}
3019
mflodmancc3d4422017-08-03 08:27:51 -07003020TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003021 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003023 DataRate::BitsPerSec(kTargetBitrateBps),
3024 DataRate::BitsPerSec(kTargetBitrateBps),
3025 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3026
3027 const int kWidth = 1280;
3028 const int kHeight = 720;
3029 int sequence = 1;
3030
3031 // Enable BALANCED preference, no initial limitation.
3032 test::FrameForwarder source;
3033 video_stream_encoder_->SetSource(&source,
3034 webrtc::DegradationPreference::BALANCED);
3035 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3036 WaitForEncodedFrame(sequence++);
3037 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3038 EXPECT_FALSE(stats.cpu_limited_resolution);
3039 EXPECT_FALSE(stats.cpu_limited_framerate);
3040 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3041
3042 // Trigger CPU overuse, should now adapt down.
3043 video_stream_encoder_->TriggerCpuOveruse();
3044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 stats = stats_proxy_->GetStats();
3047 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3048
3049 // Set new degradation preference should clear restrictions since we changed
3050 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003051 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3053 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3054 WaitForEncodedFrame(sequence++);
3055 stats = stats_proxy_->GetStats();
3056 EXPECT_FALSE(stats.cpu_limited_resolution);
3057 EXPECT_FALSE(stats.cpu_limited_framerate);
3058 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3059
3060 // Force an input frame rate to be available, or the adaptation call won't
3061 // know what framerate to adapt from.
3062 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3063 mock_stats.input_frame_rate = 30;
3064 stats_proxy_->SetMockStats(mock_stats);
3065 video_stream_encoder_->TriggerCpuOveruse();
3066 stats_proxy_->ResetMockStats();
3067 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3068 WaitForEncodedFrame(sequence++);
3069
3070 // We have now adapted once.
3071 stats = stats_proxy_->GetStats();
3072 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3073
3074 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003075 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3076 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003077 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3078 WaitForEncodedFrame(sequence++);
3079 stats = stats_proxy_->GetStats();
3080 EXPECT_FALSE(stats.cpu_limited_resolution);
3081 EXPECT_FALSE(stats.cpu_limited_framerate);
3082 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3083
3084 video_stream_encoder_->Stop();
3085}
3086
3087TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003088 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003090 DataRate::BitsPerSec(kTargetBitrateBps),
3091 DataRate::BitsPerSec(kTargetBitrateBps),
3092 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003093
asapersson0944a802017-04-07 00:57:58 -07003094 const int kWidth = 1280;
3095 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003096 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003097
asaperssonfab67072017-04-04 05:51:49 -07003098 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003099 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003100 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003101 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003102 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003103 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3104
asapersson02465b82017-04-10 01:12:52 -07003105 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003106 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003107 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003108 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003109 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003110 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003111 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003112 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3113
3114 // Set new source with adaptation still enabled.
3115 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003116 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003117 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003118
3119 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003120 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003121 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003122 stats = stats_proxy_->GetStats();
3123 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003124 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003125 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3126
sprangc5d62e22017-04-02 23:53:04 -07003127 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003128 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003129 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003130 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003131 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003132 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003133 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003134 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003135 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003136 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003137 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3138
sprangc5d62e22017-04-02 23:53:04 -07003139 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003140 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003141 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3142 mock_stats.input_frame_rate = 30;
3143 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003144 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003145 stats_proxy_->ResetMockStats();
3146
3147 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003148 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003149 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003150
3151 // Framerate now adapted.
3152 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003153 EXPECT_FALSE(stats.cpu_limited_resolution);
3154 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003155 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3156
3157 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003158 video_stream_encoder_->SetSource(&new_video_source,
3159 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003160 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003161 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003162 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003163
3164 stats = stats_proxy_->GetStats();
3165 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003166 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003167 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3168
3169 // Try to trigger overuse. Should not succeed.
3170 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003171 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003172 stats_proxy_->ResetMockStats();
3173
3174 stats = stats_proxy_->GetStats();
3175 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003176 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003177 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3178
3179 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003180 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003181 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003182 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003184 stats = stats_proxy_->GetStats();
3185 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003186 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003187 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003188
3189 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003190 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003191 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003192 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003193 stats = stats_proxy_->GetStats();
3194 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003195 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003196 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3197
3198 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003199 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003200 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003201 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003202 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003203 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003204 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003205 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003206 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003207 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003208 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3209
3210 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003211 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003212 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003213 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003214 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003215 stats = stats_proxy_->GetStats();
3216 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003217 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003218 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003219 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003220
mflodmancc3d4422017-08-03 08:27:51 -07003221 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003222}
3223
mflodmancc3d4422017-08-03 08:27:51 -07003224TEST_F(VideoStreamEncoderTest,
3225 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003226 const int kWidth = 1280;
3227 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003229 DataRate::BitsPerSec(kTargetBitrateBps),
3230 DataRate::BitsPerSec(kTargetBitrateBps),
3231 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003232
asaperssonfab67072017-04-04 05:51:49 -07003233 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003234 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003235
asaperssonfab67072017-04-04 05:51:49 -07003236 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003237 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003238
asaperssonfab67072017-04-04 05:51:49 -07003239 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003240 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003241
asaperssonfab67072017-04-04 05:51:49 -07003242 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003243 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003244
kthelgason876222f2016-11-29 01:44:11 -08003245 // Expect a scale down.
3246 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003247 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003248
asapersson02465b82017-04-10 01:12:52 -07003249 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003250 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003251 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003252 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003253
asaperssonfab67072017-04-04 05:51:49 -07003254 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003256 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003257 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003258
asaperssonfab67072017-04-04 05:51:49 -07003259 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003260 EXPECT_EQ(std::numeric_limits<int>::max(),
3261 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003262
asaperssonfab67072017-04-04 05:51:49 -07003263 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003265 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003266 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003267
asapersson02465b82017-04-10 01:12:52 -07003268 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003269 EXPECT_EQ(std::numeric_limits<int>::max(),
3270 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003271
mflodmancc3d4422017-08-03 08:27:51 -07003272 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003273}
3274
mflodmancc3d4422017-08-03 08:27:51 -07003275TEST_F(VideoStreamEncoderTest,
3276 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003277 const int kWidth = 1280;
3278 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003280 DataRate::BitsPerSec(kTargetBitrateBps),
3281 DataRate::BitsPerSec(kTargetBitrateBps),
3282 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003283
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003284 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003285 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003286 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003287 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003288
3289 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003290 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003291 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003292 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3293 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3294
3295 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003296 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003297 EXPECT_THAT(source.sink_wants(),
3298 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003299 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3300 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3301 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3302
3303 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003304 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003305 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3306 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3307 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003310}
3311
mflodmancc3d4422017-08-03 08:27:51 -07003312TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003313 const int kWidth = 1280;
3314 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003316 DataRate::BitsPerSec(kTargetBitrateBps),
3317 DataRate::BitsPerSec(kTargetBitrateBps),
3318 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003319
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003320 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003321 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003322 video_stream_encoder_->SetSource(&source,
3323 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003324 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3325 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003326 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003327
3328 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003330 EXPECT_THAT(source.sink_wants(),
3331 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3333 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3334 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3335
3336 // Trigger adapt down for same input resolution, expect no change.
3337 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3338 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003340 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3342 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3343
3344 // Trigger adapt down for larger input resolution, expect no change.
3345 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3346 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003347 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003348 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3349 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
mflodmancc3d4422017-08-03 08:27:51 -07003352 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003353}
3354
mflodmancc3d4422017-08-03 08:27:51 -07003355TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003356 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3357 const int kWidth = 640;
3358 const int kHeight = 360;
3359 const int64_t kFrameIntervalMs = 150;
3360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3361 DataRate::BitsPerSec(kTargetBitrateBps),
3362 DataRate::BitsPerSec(kTargetBitrateBps),
3363 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3364
3365 // Enable BALANCED preference, no initial limitation.
3366 AdaptingFrameForwarder source(&time_controller_);
3367 source.set_adaptation_enabled(true);
3368 video_stream_encoder_->SetSource(&source,
3369 webrtc::DegradationPreference::BALANCED);
3370
3371 int64_t timestamp_ms = kFrameIntervalMs;
3372 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3373 sink_.WaitForEncodedFrame(kWidth, kHeight);
3374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3375 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3376 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3377 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3378
3379 // Trigger adapt down, expect reduced fps (640x360@15fps).
3380 video_stream_encoder_->TriggerQualityLow();
3381 timestamp_ms += kFrameIntervalMs;
3382 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3383 sink_.WaitForEncodedFrame(timestamp_ms);
3384 EXPECT_THAT(source.sink_wants(),
3385 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3386 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3387 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3388 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 // Source requests 270p, expect reduced resolution (480x270@15fps).
3391 source.OnOutputFormatRequest(480, 270);
3392 timestamp_ms += kFrameIntervalMs;
3393 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3394 WaitForEncodedFrame(480, 270);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
3397 // Trigger adapt down, expect reduced fps (480x270@10fps).
3398 video_stream_encoder_->TriggerQualityLow();
3399 timestamp_ms += kFrameIntervalMs;
3400 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3401 sink_.WaitForEncodedFrame(timestamp_ms);
3402 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3404 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3405 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
3407 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3408 source.OnOutputFormatRequest(320, 180);
3409 timestamp_ms += kFrameIntervalMs;
3410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3411 WaitForEncodedFrame(320, 180);
3412 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3413
3414 // Trigger adapt down, expect reduced fps (320x180@7fps).
3415 video_stream_encoder_->TriggerQualityLow();
3416 timestamp_ms += kFrameIntervalMs;
3417 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3418 sink_.WaitForEncodedFrame(timestamp_ms);
3419 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3422 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Source requests VGA, expect increased resolution (640x360@7fps).
3425 source.OnOutputFormatRequest(640, 360);
3426 timestamp_ms += kFrameIntervalMs;
3427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3428 WaitForEncodedFrame(timestamp_ms);
3429 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
3431 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3432 video_stream_encoder_->TriggerQualityHigh();
3433 timestamp_ms += kFrameIntervalMs;
3434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3435 WaitForEncodedFrame(timestamp_ms);
3436 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3439 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3440
3441 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3442 video_stream_encoder_->TriggerQualityHigh();
3443 timestamp_ms += kFrameIntervalMs;
3444 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3445 WaitForEncodedFrame(timestamp_ms);
3446 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3447 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3448 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3449 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3450
3451 // Trigger adapt up, expect increased fps (640x360@maxfps).
3452 video_stream_encoder_->TriggerQualityHigh();
3453 timestamp_ms += kFrameIntervalMs;
3454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3455 WaitForEncodedFrame(timestamp_ms);
3456 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3459 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 video_stream_encoder_->Stop();
3462}
3463
3464TEST_F(VideoStreamEncoderTest,
3465 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3466 const int kWidth = 1280;
3467 const int kHeight = 720;
3468 const int64_t kFrameIntervalMs = 150;
3469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3470 DataRate::BitsPerSec(kTargetBitrateBps),
3471 DataRate::BitsPerSec(kTargetBitrateBps),
3472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3473
3474 // Enable BALANCED preference, no initial limitation.
3475 AdaptingFrameForwarder source(&time_controller_);
3476 source.set_adaptation_enabled(true);
3477 video_stream_encoder_->SetSource(&source,
3478 webrtc::DegradationPreference::BALANCED);
3479
3480 int64_t timestamp_ms = kFrameIntervalMs;
3481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3482 sink_.WaitForEncodedFrame(kWidth, kHeight);
3483 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3484 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3486 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3487
3488 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3489 video_stream_encoder_->TriggerQualityLow();
3490 timestamp_ms += kFrameIntervalMs;
3491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3492 sink_.WaitForEncodedFrame(timestamp_ms);
3493 EXPECT_THAT(source.sink_wants(),
3494 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3496 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3497 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3498
3499 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3500 video_stream_encoder_->TriggerQualityLow();
3501 timestamp_ms += kFrameIntervalMs;
3502 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3503 sink_.WaitForEncodedFrame(timestamp_ms);
3504 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3507 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3508
3509 // Trigger adapt down, expect reduced fps (640x360@15fps).
3510 video_stream_encoder_->TriggerQualityLow();
3511 timestamp_ms += kFrameIntervalMs;
3512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3513 WaitForEncodedFrame(timestamp_ms);
3514 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3517 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3518
3519 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3520 source.OnOutputFormatRequest(320, 180);
3521 timestamp_ms += kFrameIntervalMs;
3522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3523 WaitForEncodedFrame(320, 180);
3524 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3525 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3526
3527 // Trigger adapt down, expect reduced fps (320x180@7fps).
3528 video_stream_encoder_->TriggerCpuOveruse();
3529 timestamp_ms += kFrameIntervalMs;
3530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3531 WaitForEncodedFrame(timestamp_ms);
3532 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3535 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3536 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3537 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3538 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3539
3540 // Source requests HD, expect increased resolution (640x360@7fps).
3541 source.OnOutputFormatRequest(1280, 720);
3542 timestamp_ms += kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3544 WaitForEncodedFrame(timestamp_ms);
3545 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3546 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3547
3548 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3549 video_stream_encoder_->TriggerCpuUnderuse();
3550 timestamp_ms += kFrameIntervalMs;
3551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3552 WaitForEncodedFrame(timestamp_ms);
3553 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3556 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3557 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3560
3561 // Trigger adapt up, expect increased fps (640x360@maxfps).
3562 video_stream_encoder_->TriggerQualityHigh();
3563 video_stream_encoder_->TriggerCpuUnderuse();
3564 timestamp_ms += kFrameIntervalMs;
3565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3566 WaitForEncodedFrame(timestamp_ms);
3567 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3569 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3570 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3571 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3572 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3574
3575 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3576 video_stream_encoder_->TriggerQualityHigh();
3577 video_stream_encoder_->TriggerCpuUnderuse();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3585 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3586 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3590 video_stream_encoder_->TriggerQualityHigh();
3591 video_stream_encoder_->TriggerCpuUnderuse();
3592 timestamp_ms += kFrameIntervalMs;
3593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3594 WaitForEncodedFrame(timestamp_ms);
3595 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3600 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3601 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3602
3603 video_stream_encoder_->Stop();
3604}
3605
3606TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003607 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003608 const int kWidth = 1280;
3609 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003611 DataRate::BitsPerSec(kTargetBitrateBps),
3612 DataRate::BitsPerSec(kTargetBitrateBps),
3613 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003614
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003615 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003616 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003617 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003618 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003619
3620 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003621 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003623 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3624 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3625
3626 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003627 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003628 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3630 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3631
mflodmancc3d4422017-08-03 08:27:51 -07003632 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003633}
3634
mflodmancc3d4422017-08-03 08:27:51 -07003635TEST_F(VideoStreamEncoderTest,
3636 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003637 const int kWidth = 1280;
3638 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003640 DataRate::BitsPerSec(kTargetBitrateBps),
3641 DataRate::BitsPerSec(kTargetBitrateBps),
3642 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003643
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003644 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003645 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003646 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003647 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003648
3649 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003650 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003651 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003653 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3654
3655 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003656 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003657 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003658 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003659 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3660
mflodmancc3d4422017-08-03 08:27:51 -07003661 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003662}
3663
mflodmancc3d4422017-08-03 08:27:51 -07003664TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003665 const int kWidth = 1280;
3666 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003668 DataRate::BitsPerSec(kTargetBitrateBps),
3669 DataRate::BitsPerSec(kTargetBitrateBps),
3670 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003671
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003672 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003673 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003674 video_stream_encoder_->SetSource(&source,
3675 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003676
3677 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3678 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3681 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3683
3684 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003685 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003687 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3689 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3690
mflodmancc3d4422017-08-03 08:27:51 -07003691 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003692}
3693
mflodmancc3d4422017-08-03 08:27:51 -07003694TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003695 const int kWidth = 1280;
3696 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003698 DataRate::BitsPerSec(kTargetBitrateBps),
3699 DataRate::BitsPerSec(kTargetBitrateBps),
3700 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003701
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003702 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003703 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003704 video_stream_encoder_->SetSource(&source,
3705 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003706
3707 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3708 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3712 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3713
3714 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003715 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003716 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003717 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3720
mflodmancc3d4422017-08-03 08:27:51 -07003721 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003722}
3723
mflodmancc3d4422017-08-03 08:27:51 -07003724TEST_F(VideoStreamEncoderTest,
3725 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003726 const int kWidth = 1280;
3727 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003729 DataRate::BitsPerSec(kTargetBitrateBps),
3730 DataRate::BitsPerSec(kTargetBitrateBps),
3731 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003732
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003733 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003734 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003735 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003736 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003737 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003738
3739 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003740 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003741 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3743 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3744
3745 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003746 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003747 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003748 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003749 EXPECT_THAT(source.sink_wants(),
3750 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003751 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
3754 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003755 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003756 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003757 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3758 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3759 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3760
mflodmancc3d4422017-08-03 08:27:51 -07003761 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003762}
3763
mflodmancc3d4422017-08-03 08:27:51 -07003764TEST_F(VideoStreamEncoderTest,
3765 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003766 const int kWidth = 1280;
3767 const int kHeight = 720;
3768 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003770 DataRate::BitsPerSec(kTargetBitrateBps),
3771 DataRate::BitsPerSec(kTargetBitrateBps),
3772 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003773
3774 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3775 stats.input_frame_rate = kInputFps;
3776 stats_proxy_->SetMockStats(stats);
3777
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003778 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003779 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3780 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003781 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003782
3783 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003784 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003785 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3786 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003787 EXPECT_THAT(video_source_.sink_wants(),
3788 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003789
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003790 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003791 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003792 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003793 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003794 // Give the encoder queue time to process the change in degradation preference
3795 // by waiting for an encoded frame.
3796 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3797 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003798 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003799
3800 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003801 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003802 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3803 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003804 EXPECT_THAT(new_video_source.sink_wants(),
3805 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003806
3807 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003808 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003809 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003810
mflodmancc3d4422017-08-03 08:27:51 -07003811 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003812}
3813
mflodmancc3d4422017-08-03 08:27:51 -07003814TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003815 const int kWidth = 1280;
3816 const int kHeight = 720;
3817 const size_t kNumFrames = 10;
3818
Henrik Boström381d1092020-05-12 18:49:07 +02003819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003820 DataRate::BitsPerSec(kTargetBitrateBps),
3821 DataRate::BitsPerSec(kTargetBitrateBps),
3822 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003823
asaperssond0de2952017-04-21 01:47:31 -07003824 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003825 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003826 video_source_.set_adaptation_enabled(true);
3827
3828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3830
3831 int downscales = 0;
3832 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003833 video_source_.IncomingCapturedFrame(
3834 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3835 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003836
asaperssonfab67072017-04-04 05:51:49 -07003837 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003838 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003839 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003840 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003841
3842 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3843 ++downscales;
3844
3845 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3846 EXPECT_EQ(downscales,
3847 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3848 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003849 }
mflodmancc3d4422017-08-03 08:27:51 -07003850 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003851}
3852
mflodmancc3d4422017-08-03 08:27:51 -07003853TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003854 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3855 const int kWidth = 1280;
3856 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003858 DataRate::BitsPerSec(kTargetBitrateBps),
3859 DataRate::BitsPerSec(kTargetBitrateBps),
3860 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003861
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003862 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003863 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003864 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003865 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003866 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003867
Åsa Persson8c1bf952018-09-13 10:42:19 +02003868 int64_t timestamp_ms = kFrameIntervalMs;
3869 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003870 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003871 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3873 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3874
3875 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003876 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003877 timestamp_ms += kFrameIntervalMs;
3878 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3879 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003880 EXPECT_THAT(source.sink_wants(),
3881 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003882 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3884
3885 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003886 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003887 timestamp_ms += kFrameIntervalMs;
3888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003889 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003890 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003891 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3892 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3893
3894 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003895 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3898 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source.sink_wants(),
3900 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003901 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3902 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3903
3904 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003905 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003908 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003909 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3911 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3912
mflodmancc3d4422017-08-03 08:27:51 -07003913 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003914}
3915
mflodmancc3d4422017-08-03 08:27:51 -07003916TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003917 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3918 const int kWidth = 1280;
3919 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003921 DataRate::BitsPerSec(kTargetBitrateBps),
3922 DataRate::BitsPerSec(kTargetBitrateBps),
3923 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003924
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003925 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003926 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003927 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003928 video_stream_encoder_->SetSource(&source,
3929 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003930
Åsa Persson8c1bf952018-09-13 10:42:19 +02003931 int64_t timestamp_ms = kFrameIntervalMs;
3932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003933 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003935 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3936 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3937
3938 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003939 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003940 timestamp_ms += kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3942 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003943 EXPECT_THAT(source.sink_wants(),
3944 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003945 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3946 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3947
3948 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003949 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003950 timestamp_ms += kFrameIntervalMs;
3951 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003952 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003953 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003954 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3955 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3956
3957 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003958 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3961 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(),
3963 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003964 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3965 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3966
3967 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003968 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003969 timestamp_ms += kFrameIntervalMs;
3970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003971 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003972 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003973 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3974 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3975
mflodmancc3d4422017-08-03 08:27:51 -07003976 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003977}
3978
Sergey Silkin41c650b2019-10-14 13:12:19 +02003979TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3980 fake_encoder_.SetResolutionBitrateLimits(
3981 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3982
Henrik Boström381d1092020-05-12 18:49:07 +02003983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003984 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3985 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3986 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3987 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003988
3989 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003990 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003991 source.set_adaptation_enabled(true);
3992 video_stream_encoder_->SetSource(
3993 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3994
3995 // Insert 720p frame.
3996 int64_t timestamp_ms = kFrameIntervalMs;
3997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3998 WaitForEncodedFrame(1280, 720);
3999
4000 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004002 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4003 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4004 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4005 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004006 video_stream_encoder_->TriggerQualityLow();
4007
4008 // Insert 720p frame. It should be downscaled and encoded.
4009 timestamp_ms += kFrameIntervalMs;
4010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4011 WaitForEncodedFrame(960, 540);
4012
4013 // Trigger adapt up. Higher resolution should not be requested duo to lack
4014 // of bitrate.
4015 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004016 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004017
4018 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004020 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4021 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4022 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4023 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004024
4025 // Trigger adapt up. Higher resolution should be requested.
4026 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004027 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004028
4029 video_stream_encoder_->Stop();
4030}
4031
4032TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4033 fake_encoder_.SetResolutionBitrateLimits(
4034 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4035
4036 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004038 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4039 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4040 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4041 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004042
4043 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004044 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004045 source.set_adaptation_enabled(true);
4046 video_stream_encoder_->SetSource(
4047 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4048
4049 // Insert 720p frame. It should be dropped and lower resolution should be
4050 // requested.
4051 int64_t timestamp_ms = kFrameIntervalMs;
4052 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4053 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004054 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004055
4056 // Insert 720p frame. It should be downscaled and encoded.
4057 timestamp_ms += kFrameIntervalMs;
4058 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4059 WaitForEncodedFrame(960, 540);
4060
4061 video_stream_encoder_->Stop();
4062}
4063
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004064class BalancedDegradationTest : public VideoStreamEncoderTest {
4065 protected:
4066 void SetupTest() {
4067 // Reset encoder for field trials to take effect.
4068 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02004069 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004070
4071 // Enable BALANCED preference.
4072 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004073 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4074 }
4075
4076 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02004077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004078 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
4079 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004080 }
4081
Åsa Persson45b176f2019-09-30 11:19:05 +02004082 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004083 timestamp_ms_ += kFrameIntervalMs;
4084 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004085 }
4086
4087 void InsertFrameAndWaitForEncoded() {
4088 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004089 sink_.WaitForEncodedFrame(timestamp_ms_);
4090 }
4091
4092 const int kWidth = 640; // pixels:640x360=230400
4093 const int kHeight = 360;
4094 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4095 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004096 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004097};
4098
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004099TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004100 test::ScopedFieldTrials field_trials(
4101 "WebRTC-Video-BalancedDegradationSettings/"
4102 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4103 SetupTest();
4104
4105 // Force input frame rate.
4106 const int kInputFps = 24;
4107 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4108 stats.input_frame_rate = kInputFps;
4109 stats_proxy_->SetMockStats(stats);
4110
Åsa Persson45b176f2019-09-30 11:19:05 +02004111 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004112 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004113
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004114 // Trigger adapt down, expect scaled down framerate and resolution,
4115 // since Fps diff (input-requested:0) < threshold.
4116 video_stream_encoder_->TriggerQualityLow();
4117 EXPECT_THAT(source_.sink_wants(),
4118 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004119
4120 video_stream_encoder_->Stop();
4121}
4122
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004123TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004124 test::ScopedFieldTrials field_trials(
4125 "WebRTC-Video-BalancedDegradationSettings/"
4126 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4127 SetupTest();
4128
4129 // Force input frame rate.
4130 const int kInputFps = 25;
4131 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4132 stats.input_frame_rate = kInputFps;
4133 stats_proxy_->SetMockStats(stats);
4134
Åsa Persson45b176f2019-09-30 11:19:05 +02004135 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004136 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004137
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004138 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4139 // Fps diff (input-requested:1) == threshold.
4140 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004141 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004142
4143 video_stream_encoder_->Stop();
4144}
4145
4146TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4147 test::ScopedFieldTrials field_trials(
4148 "WebRTC-Video-BalancedDegradationSettings/"
4149 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4150 SetupTest();
4151
4152 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4153
Åsa Persson45b176f2019-09-30 11:19:05 +02004154 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004155 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004156
4157 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4158 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004159 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004160
4161 video_stream_encoder_->Stop();
4162}
4163
Åsa Perssonccfb3402019-09-25 15:13:04 +02004164TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004165 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004166 "WebRTC-Video-BalancedDegradationSettings/"
4167 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004168 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004169
Åsa Persson1b247f12019-08-14 17:26:39 +02004170 const int kMinBitrateBps = 425000;
4171 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004172 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02004173
Åsa Persson45b176f2019-09-30 11:19:05 +02004174 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004175 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004176 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4177
4178 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4179 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004180 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004181 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004182 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4183
4184 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4185 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004186 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004187 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004188 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4189
Åsa Persson30ab0152019-08-27 12:22:33 +02004190 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4191 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004192 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004193 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004194 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004195 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4196
4197 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004198 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004199 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004200 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004201
Åsa Persson30ab0152019-08-27 12:22:33 +02004202 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004203 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02004204 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004205 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004206 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004207 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4208
4209 video_stream_encoder_->Stop();
4210}
4211
Åsa Perssonccfb3402019-09-25 15:13:04 +02004212TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004213 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4214 test::ScopedFieldTrials field_trials(
4215 "WebRTC-Video-BalancedDegradationSettings/"
4216 "pixels:57600|129600|230400,fps:7|24|24/");
4217 SetupTest();
4218 OnBitrateUpdated(kLowTargetBitrateBps);
4219
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004220 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004221
4222 // Insert frame, expect scaled down:
4223 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4224 InsertFrame();
4225 EXPECT_FALSE(WaitForFrame(1000));
4226 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4227 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4228
4229 // Insert frame, expect scaled down:
4230 // resolution (320x180@24fps).
4231 InsertFrame();
4232 EXPECT_FALSE(WaitForFrame(1000));
4233 EXPECT_LT(source_.sink_wants().max_pixel_count,
4234 source_.last_wants().max_pixel_count);
4235 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4236
4237 // Frame should not be dropped (min pixels per frame reached).
4238 InsertFrameAndWaitForEncoded();
4239
4240 video_stream_encoder_->Stop();
4241}
4242
4243TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004244 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004245 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004246 "WebRTC-Video-BalancedDegradationSettings/"
4247 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004248 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004249
Åsa Persson30ab0152019-08-27 12:22:33 +02004250 const int kResolutionMinBitrateBps = 435000;
4251 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004252 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004253
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004256 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4259 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004260 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004261 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
4264 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4265 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004266 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004267 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004268 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4269
4270 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4271 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004272 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004273 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004274 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4275
Åsa Persson30ab0152019-08-27 12:22:33 +02004276 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4277 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004279 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004280 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4281
4282 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4283 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004284 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004285 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4286
4287 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004288 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004289 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004290 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004291 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004292 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4293
4294 video_stream_encoder_->Stop();
4295}
4296
Åsa Perssonccfb3402019-09-25 15:13:04 +02004297TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004299 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004300 "WebRTC-Video-BalancedDegradationSettings/"
4301 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004302 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004303
Åsa Persson30ab0152019-08-27 12:22:33 +02004304 const int kMinBitrateBps = 425000;
4305 const int kTooLowMinBitrateBps = 424000;
4306 const int kResolutionMinBitrateBps = 435000;
4307 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004308 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004309
Åsa Persson45b176f2019-09-30 11:19:05 +02004310 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004311 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
4314 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4315 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004317 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004323 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004324 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004329 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004330 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
4332 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4333 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004334 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004335 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004338 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004340 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004341 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004342 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004345 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004347 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004348 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4349
4350 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004351 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004352 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004353 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004354 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
Åsa Persson1b247f12019-08-14 17:26:39 +02004357 video_stream_encoder_->Stop();
4358}
4359
mflodmancc3d4422017-08-03 08:27:51 -07004360TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004361 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4362 const int kWidth = 1280;
4363 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004365 DataRate::BitsPerSec(kTargetBitrateBps),
4366 DataRate::BitsPerSec(kTargetBitrateBps),
4367 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004368
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004369 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004370 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004371 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004373 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004374
Åsa Persson8c1bf952018-09-13 10:42:19 +02004375 int64_t timestamp_ms = kFrameIntervalMs;
4376 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004377 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004378 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004379 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4380 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4382 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4383
4384 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004385 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004386 timestamp_ms += kFrameIntervalMs;
4387 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4388 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004389 EXPECT_THAT(source.sink_wants(),
4390 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004391 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4392 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4393 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4394 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4395
4396 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004397 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004398 timestamp_ms += kFrameIntervalMs;
4399 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4400 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004401 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004402 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4403 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4404 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4405 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4406
Jonathan Yubc771b72017-12-08 17:04:29 -08004407 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004408 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004409 timestamp_ms += kFrameIntervalMs;
4410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4411 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004412 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004413 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4417
Jonathan Yubc771b72017-12-08 17:04:29 -08004418 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004419 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004420 timestamp_ms += kFrameIntervalMs;
4421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4422 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004423 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004424 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004425 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4426 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4427 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4428 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4429
Jonathan Yubc771b72017-12-08 17:04:29 -08004430 // Trigger quality adapt down, expect no change (min resolution reached).
4431 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004432 timestamp_ms += kFrameIntervalMs;
4433 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4434 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004435 EXPECT_THAT(source.sink_wants(), FpsMax());
4436 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004437 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4438 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4439 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4440 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4441
Evan Shrubsole64469032020-06-11 10:45:29 +02004442 // Trigger quality adapt up, expect upscaled resolution (480x270).
4443 video_stream_encoder_->TriggerQualityHigh();
4444 timestamp_ms += kFrameIntervalMs;
4445 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4446 WaitForEncodedFrame(timestamp_ms);
4447 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4448 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4449 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4450 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4451 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4452
4453 // Trigger quality and cpu adapt up since both are most limited, expect
4454 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004455 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004456 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004457 timestamp_ms += kFrameIntervalMs;
4458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4459 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004460 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004461 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4462 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4463 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004464 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004465
Evan Shrubsole64469032020-06-11 10:45:29 +02004466 // Trigger quality and cpu adapt up since both are most limited, expect
4467 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004468 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004469 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004470 timestamp_ms += kFrameIntervalMs;
4471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004473 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004474 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004475 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004477 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4478 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004479
Evan Shrubsole64469032020-06-11 10:45:29 +02004480 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4481 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004482 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004486 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004487 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4488 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004489 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004490 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004491
4492 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004493 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004494 timestamp_ms += kFrameIntervalMs;
4495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004496 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004499 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004501 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004502 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004503
mflodmancc3d4422017-08-03 08:27:51 -07004504 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004505}
4506
mflodmancc3d4422017-08-03 08:27:51 -07004507TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004508 const int kWidth = 640;
4509 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004510
Henrik Boström381d1092020-05-12 18:49:07 +02004511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004512 DataRate::BitsPerSec(kTargetBitrateBps),
4513 DataRate::BitsPerSec(kTargetBitrateBps),
4514 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004515
perkj803d97f2016-11-01 11:45:46 -07004516 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004517 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004518 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004519 }
4520
mflodmancc3d4422017-08-03 08:27:51 -07004521 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004522 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004523 video_source_.IncomingCapturedFrame(CreateFrame(
4524 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004525 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004526 }
4527
mflodmancc3d4422017-08-03 08:27:51 -07004528 video_stream_encoder_->Stop();
4529 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004530 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004531
Ying Wangef3998f2019-12-09 13:06:53 +01004532 EXPECT_METRIC_EQ(
4533 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4534 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004535 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4536}
4537
mflodmancc3d4422017-08-03 08:27:51 -07004538TEST_F(VideoStreamEncoderTest,
4539 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004541 DataRate::BitsPerSec(kTargetBitrateBps),
4542 DataRate::BitsPerSec(kTargetBitrateBps),
4543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004544 const int kWidth = 640;
4545 const int kHeight = 360;
4546
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004547 video_stream_encoder_->SetSource(&video_source_,
4548 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004549
4550 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4551 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004552 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004553 }
4554
mflodmancc3d4422017-08-03 08:27:51 -07004555 video_stream_encoder_->Stop();
4556 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004557 stats_proxy_.reset();
4558
4559 EXPECT_EQ(0,
4560 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4561}
4562
Per Kjellanderdcef6412020-10-07 15:09:05 +02004563TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4564 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004565 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004566 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004567
4568 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004569 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004570 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004571 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4572 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004573
Henrik Boström381d1092020-05-12 18:49:07 +02004574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004575 DataRate::BitsPerSec(kLowTargetBitrateBps),
4576 DataRate::BitsPerSec(kLowTargetBitrateBps),
4577 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004578
sprang57c2fff2017-01-16 06:24:02 -08004579 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004580 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4581 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004582 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4583 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4584
Erik Språngd7329ca2019-02-21 21:19:53 +01004585 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004586 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004587 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004588
Per Kjellanderdcef6412020-10-07 15:09:05 +02004589 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004590 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004591 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4592 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004593 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004594 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004595
Per Kjellanderdcef6412020-10-07 15:09:05 +02004596 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004597 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004598 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004599 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004600 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4601 WaitForEncodedFrame(CurrentTimeMs());
4602 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004603 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004604 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004605
mflodmancc3d4422017-08-03 08:27:51 -07004606 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004607}
4608
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004609TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004610 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004611 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004612 kVideoLayersAllocation);
4613
4614 const int kDefaultFps = 30;
4615
4616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4617 DataRate::BitsPerSec(kLowTargetBitrateBps),
4618 DataRate::BitsPerSec(kLowTargetBitrateBps),
4619 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4620
4621 video_source_.IncomingCapturedFrame(
4622 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4623 WaitForEncodedFrame(CurrentTimeMs());
4624 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4625 VideoLayersAllocation last_layer_allocation =
4626 sink_.GetLastVideoLayersAllocation();
4627 // kLowTargetBitrateBps is only enough for one spatial layer.
4628 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4629
4630 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004631 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004632 // Check that encoder has been updated too, not just allocation observer.
4633 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4634 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4635
Erik Språng9d69cbe2020-10-22 17:44:42 +02004636 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004637 int number_of_layers_allocation = 1;
4638 const int64_t start_time_ms = CurrentTimeMs();
4639 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4640 video_source_.IncomingCapturedFrame(
4641 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4642 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004643 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4644 number_of_layers_allocation = sink_.number_of_layers_allocations();
4645 VideoLayersAllocation new_allocation =
4646 sink_.GetLastVideoLayersAllocation();
4647 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4648 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4649 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4650 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4651 .target_bitrate_per_temporal_layer,
4652 last_layer_allocation.active_spatial_layers[0]
4653 .target_bitrate_per_temporal_layer);
4654 last_layer_allocation = new_allocation;
4655 }
4656 }
4657 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4658 video_stream_encoder_->Stop();
4659}
4660
4661TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004662 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004663 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4664 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4665 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666 VideoEncoderConfig video_encoder_config;
4667 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4668 /* num_streams*/ 3, &video_encoder_config);
4669 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4670 video_encoder_config.content_type =
4671 VideoEncoderConfig::ContentType::kRealtimeVideo;
4672 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004673 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004674 VideoEncoder::GetDefaultVp8Settings());
4675 for (auto& layer : video_encoder_config.simulcast_layers) {
4676 layer.num_temporal_layers = 2;
4677 }
4678 // Simulcast layers are used for enabling/disabling streams.
4679 video_encoder_config.simulcast_layers[0].active = true;
4680 video_encoder_config.simulcast_layers[1].active = false;
4681 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004682 ConfigureEncoder(std::move(video_encoder_config),
4683 VideoStreamEncoder::BitrateAllocationCallbackType::
4684 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004685
4686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4687 DataRate::BitsPerSec(kTargetBitrateBps),
4688 DataRate::BitsPerSec(kTargetBitrateBps),
4689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4690
4691 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4692 WaitForEncodedFrame(CurrentTimeMs());
4693 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4694 VideoLayersAllocation last_layer_allocation =
4695 sink_.GetLastVideoLayersAllocation();
4696
4697 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4698 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4699 .target_bitrate_per_temporal_layer,
4700 SizeIs(2));
4701 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4702 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4703 video_stream_encoder_->Stop();
4704}
4705
4706TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004707 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004708 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4709 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4710 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004711 VideoEncoderConfig video_encoder_config;
4712 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4713 /* num_streams*/ 3, &video_encoder_config);
4714 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4715 video_encoder_config.content_type =
4716 VideoEncoderConfig::ContentType::kRealtimeVideo;
4717 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004718 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004719 VideoEncoder::GetDefaultVp8Settings());
4720 for (auto& layer : video_encoder_config.simulcast_layers) {
4721 layer.num_temporal_layers = 2;
4722 }
4723 // Simulcast layers are used for enabling/disabling streams.
4724 video_encoder_config.simulcast_layers[0].active = true;
4725 video_encoder_config.simulcast_layers[1].active = false;
4726 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004727 ConfigureEncoder(std::move(video_encoder_config),
4728 VideoStreamEncoder::BitrateAllocationCallbackType::
4729 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004730
4731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4732 DataRate::BitsPerSec(kTargetBitrateBps),
4733 DataRate::BitsPerSec(kTargetBitrateBps),
4734 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4735
4736 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4737 WaitForEncodedFrame(CurrentTimeMs());
4738 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4739 VideoLayersAllocation last_layer_allocation =
4740 sink_.GetLastVideoLayersAllocation();
4741
4742 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4743 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4744 .target_bitrate_per_temporal_layer,
4745 SizeIs(2));
4746 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4747
4748 video_stream_encoder_->Stop();
4749}
4750
4751TEST_F(VideoStreamEncoderTest,
4752 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4753 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4754 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755 VideoEncoderConfig video_encoder_config;
4756 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4757 /* num_streams*/ 1, &video_encoder_config);
4758 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4759 video_encoder_config.content_type =
4760 VideoEncoderConfig::ContentType::kRealtimeVideo;
4761 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4762 vp9_settings.numberOfSpatialLayers = 2;
4763 vp9_settings.numberOfTemporalLayers = 2;
4764 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4765 vp9_settings.automaticResizeOn = false;
4766 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004767 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004768 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004769 ConfigureEncoder(std::move(video_encoder_config),
4770 VideoStreamEncoder::BitrateAllocationCallbackType::
4771 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772
4773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4774 DataRate::BitsPerSec(kTargetBitrateBps),
4775 DataRate::BitsPerSec(kTargetBitrateBps),
4776 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4777
4778 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4779 WaitForEncodedFrame(CurrentTimeMs());
4780 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4781 VideoLayersAllocation last_layer_allocation =
4782 sink_.GetLastVideoLayersAllocation();
4783
4784 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4785 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4786 .target_bitrate_per_temporal_layer,
4787 SizeIs(2));
4788 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4789 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4790 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4791 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4792 .target_bitrate_per_temporal_layer,
4793 SizeIs(2));
4794 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4796 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4797
4798 // Since full SVC is used, expect the top layer to utilize the full target
4799 // rate.
4800 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4801 .target_bitrate_per_temporal_layer[1],
4802 DataRate::BitsPerSec(kTargetBitrateBps));
4803 video_stream_encoder_->Stop();
4804}
4805
4806TEST_F(VideoStreamEncoderTest,
4807 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4808 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4809 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004810 VideoEncoderConfig video_encoder_config;
4811 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4812 /* num_streams*/ 1, &video_encoder_config);
4813 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4814 video_encoder_config.content_type =
4815 VideoEncoderConfig::ContentType::kRealtimeVideo;
4816 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4817 vp9_settings.numberOfSpatialLayers = 2;
4818 vp9_settings.numberOfTemporalLayers = 2;
4819 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4820 vp9_settings.automaticResizeOn = false;
4821 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004822 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004824 ConfigureEncoder(std::move(video_encoder_config),
4825 VideoStreamEncoder::BitrateAllocationCallbackType::
4826 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004827
4828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4829 DataRate::BitsPerSec(kTargetBitrateBps),
4830 DataRate::BitsPerSec(kTargetBitrateBps),
4831 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4832
4833 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4834 WaitForEncodedFrame(CurrentTimeMs());
4835 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4836 VideoLayersAllocation last_layer_allocation =
4837 sink_.GetLastVideoLayersAllocation();
4838
4839 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4840 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4841 .target_bitrate_per_temporal_layer,
4842 SizeIs(1));
4843 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4844 .target_bitrate_per_temporal_layer,
4845 SizeIs(1));
4846 // Since full SVC is used, expect the top layer to utilize the full target
4847 // rate.
4848 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4849 .target_bitrate_per_temporal_layer[0],
4850 DataRate::BitsPerSec(kTargetBitrateBps));
4851 video_stream_encoder_->Stop();
4852}
4853
4854TEST_F(VideoStreamEncoderTest,
4855 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4856 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4857 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004858 VideoEncoderConfig video_encoder_config;
4859 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4860 /* num_streams*/ 1, &video_encoder_config);
4861 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4862 video_encoder_config.content_type =
4863 VideoEncoderConfig::ContentType::kRealtimeVideo;
4864 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4865 vp9_settings.numberOfSpatialLayers = 2;
4866 vp9_settings.numberOfTemporalLayers = 2;
4867 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4868 vp9_settings.automaticResizeOn = false;
4869 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004870 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004871 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004872 ConfigureEncoder(std::move(video_encoder_config),
4873 VideoStreamEncoder::BitrateAllocationCallbackType::
4874 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004875
4876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4877 DataRate::BitsPerSec(kTargetBitrateBps),
4878 DataRate::BitsPerSec(kTargetBitrateBps),
4879 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4880
4881 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4882 WaitForEncodedFrame(CurrentTimeMs());
4883 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4884 VideoLayersAllocation last_layer_allocation =
4885 sink_.GetLastVideoLayersAllocation();
4886
4887 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4888 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4889 .target_bitrate_per_temporal_layer,
4890 SizeIs(2));
4891 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4892 .target_bitrate_per_temporal_layer,
4893 SizeIs(2));
4894 // Since KSVC is, spatial layers are independend except on key frames.
4895 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4896 .target_bitrate_per_temporal_layer[1],
4897 DataRate::BitsPerSec(kTargetBitrateBps));
4898 video_stream_encoder_->Stop();
4899}
4900
4901TEST_F(VideoStreamEncoderTest,
4902 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4903 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4904 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004906 VideoEncoderConfig video_encoder_config;
4907 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4908 /* num_streams*/ 1, &video_encoder_config);
4909 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4910 video_encoder_config.content_type =
4911 VideoEncoderConfig::ContentType::kRealtimeVideo;
4912 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4913 vp9_settings.numberOfSpatialLayers = 3;
4914 vp9_settings.numberOfTemporalLayers = 2;
4915 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4916 vp9_settings.automaticResizeOn = false;
4917 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004918 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004919 vp9_settings);
4920 // Simulcast layers are used for enabling/disabling streams.
4921 video_encoder_config.simulcast_layers.resize(3);
4922 video_encoder_config.simulcast_layers[0].active = false;
4923 video_encoder_config.simulcast_layers[1].active = true;
4924 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004925 ConfigureEncoder(std::move(video_encoder_config),
4926 VideoStreamEncoder::BitrateAllocationCallbackType::
4927 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004928
4929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4930 DataRate::BitsPerSec(kTargetBitrateBps),
4931 DataRate::BitsPerSec(kTargetBitrateBps),
4932 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4933
4934 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4935 WaitForEncodedFrame(CurrentTimeMs());
4936 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4937 VideoLayersAllocation last_layer_allocation =
4938 sink_.GetLastVideoLayersAllocation();
4939
4940 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4941 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4942 .target_bitrate_per_temporal_layer,
4943 SizeIs(2));
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4946
4947 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4948 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4949 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4950 .target_bitrate_per_temporal_layer,
4951 SizeIs(2));
4952 // Since full SVC is used, expect the top layer to utilize the full target
4953 // rate.
4954 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4955 .target_bitrate_per_temporal_layer[1],
4956 DataRate::BitsPerSec(kTargetBitrateBps));
4957 video_stream_encoder_->Stop();
4958}
4959
4960TEST_F(VideoStreamEncoderTest,
4961 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4962 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4963 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4964 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004965 VideoEncoderConfig video_encoder_config;
4966 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4967 /* num_streams*/ 1, &video_encoder_config);
4968 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4969 video_encoder_config.content_type =
4970 VideoEncoderConfig::ContentType::kRealtimeVideo;
4971 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4972 vp9_settings.numberOfSpatialLayers = 3;
4973 vp9_settings.numberOfTemporalLayers = 2;
4974 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4975 vp9_settings.automaticResizeOn = false;
4976 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004977 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004978 vp9_settings);
4979 // Simulcast layers are used for enabling/disabling streams.
4980 video_encoder_config.simulcast_layers.resize(3);
4981 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004982 ConfigureEncoder(std::move(video_encoder_config),
4983 VideoStreamEncoder::BitrateAllocationCallbackType::
4984 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004985
4986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4987 DataRate::BitsPerSec(kTargetBitrateBps),
4988 DataRate::BitsPerSec(kTargetBitrateBps),
4989 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4990
4991 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4992 WaitForEncodedFrame(CurrentTimeMs());
4993 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4994 VideoLayersAllocation last_layer_allocation =
4995 sink_.GetLastVideoLayersAllocation();
4996
4997 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4998 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4999 .target_bitrate_per_temporal_layer,
5000 SizeIs(2));
5001 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5003
5004 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5005 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5006 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5007 .target_bitrate_per_temporal_layer,
5008 SizeIs(2));
5009 video_stream_encoder_->Stop();
5010}
5011
5012TEST_F(VideoStreamEncoderTest,
5013 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5014 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5015 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5016 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005017 VideoEncoderConfig video_encoder_config;
5018 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5019 /* num_streams*/ 1, &video_encoder_config);
5020 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
5021 video_encoder_config.content_type =
5022 VideoEncoderConfig::ContentType::kRealtimeVideo;
5023 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5024 vp9_settings.numberOfSpatialLayers = 3;
5025 vp9_settings.numberOfTemporalLayers = 2;
5026 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5027 vp9_settings.automaticResizeOn = false;
5028 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005029 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005030 vp9_settings);
5031 // Simulcast layers are used for enabling/disabling streams.
5032 video_encoder_config.simulcast_layers.resize(3);
5033 video_encoder_config.simulcast_layers[0].active = false;
5034 video_encoder_config.simulcast_layers[1].active = false;
5035 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005036 ConfigureEncoder(std::move(video_encoder_config),
5037 VideoStreamEncoder::BitrateAllocationCallbackType::
5038 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005039
5040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5041 DataRate::BitsPerSec(kTargetBitrateBps),
5042 DataRate::BitsPerSec(kTargetBitrateBps),
5043 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5044
5045 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5046 WaitForEncodedFrame(CurrentTimeMs());
5047 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5048 VideoLayersAllocation last_layer_allocation =
5049 sink_.GetLastVideoLayersAllocation();
5050
5051 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5052 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5053 .target_bitrate_per_temporal_layer,
5054 SizeIs(2));
5055 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5056 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5057 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5058 .target_bitrate_per_temporal_layer[1],
5059 DataRate::BitsPerSec(kTargetBitrateBps));
5060 video_stream_encoder_->Stop();
5061}
5062
5063TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5064 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005065 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005066 kVideoLayersAllocation);
5067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5068 DataRate::BitsPerSec(kTargetBitrateBps),
5069 DataRate::BitsPerSec(kTargetBitrateBps),
5070 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5071
5072 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5073 WaitForEncodedFrame(CurrentTimeMs());
5074 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5075 VideoLayersAllocation last_layer_allocation =
5076 sink_.GetLastVideoLayersAllocation();
5077
5078 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5079 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5080 .target_bitrate_per_temporal_layer,
5081 SizeIs(1));
5082 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5083 .target_bitrate_per_temporal_layer[0],
5084 DataRate::BitsPerSec(kTargetBitrateBps));
5085 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5086 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5087 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5088 video_stream_encoder_->Stop();
5089}
5090
5091TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005092 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5093 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005094 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005095 kVideoLayersAllocation);
5096
5097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5098 DataRate::BitsPerSec(kLowTargetBitrateBps),
5099 DataRate::BitsPerSec(kLowTargetBitrateBps),
5100 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5101
5102 video_source_.IncomingCapturedFrame(
5103 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5104 WaitForEncodedFrame(CurrentTimeMs());
5105 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5106 VideoLayersAllocation last_layer_allocation =
5107 sink_.GetLastVideoLayersAllocation();
5108 // kLowTargetBitrateBps is only enough for one spatial layer.
5109 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5110 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5111 .target_bitrate_per_temporal_layer[0],
5112 DataRate::BitsPerSec(kLowTargetBitrateBps));
5113
5114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5115 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5116 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5117 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5118 video_source_.IncomingCapturedFrame(
5119 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5120 WaitForEncodedFrame(CurrentTimeMs());
5121
5122 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5123 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5124 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5125 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5126 .target_bitrate_per_temporal_layer[0],
5127 DataRate::Zero());
5128
5129 video_stream_encoder_->Stop();
5130}
5131
Per Kjellander4190ce92020-12-15 17:24:55 +01005132TEST_F(VideoStreamEncoderTest,
5133 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5134 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005135 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005136 kVideoLayersAllocation);
5137
5138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5139 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5140 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5141 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5142
5143 video_source_.IncomingCapturedFrame(
5144 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5145 WaitForEncodedFrame(CurrentTimeMs());
5146 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5147 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5148 SizeIs(2));
5149 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5150 codec_width_);
5151 EXPECT_EQ(
5152 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5153 codec_height_);
5154
5155 video_source_.IncomingCapturedFrame(
5156 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5157 WaitForEncodedFrame(CurrentTimeMs());
5158 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5159 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5160 SizeIs(2));
5161 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5162 codec_width_ / 2);
5163 EXPECT_EQ(
5164 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5165 codec_height_ / 2);
5166
5167 video_stream_encoder_->Stop();
5168}
5169
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005170TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5171 // 2 TLs configured, temporal layers supported by encoder.
5172 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005173 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005174 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005175 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005176 fake_encoder_.SetTemporalLayersSupported(0, true);
5177
5178 // Bitrate allocated across temporal layers.
5179 const int kTl0Bps = kTargetBitrateBps *
5180 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005181 kNumTemporalLayers, /*temporal_id*/ 0,
5182 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005183 const int kTl1Bps = kTargetBitrateBps *
5184 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005185 kNumTemporalLayers, /*temporal_id*/ 1,
5186 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005187 VideoBitrateAllocation expected_bitrate;
5188 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5189 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5190
5191 VerifyAllocatedBitrate(expected_bitrate);
5192 video_stream_encoder_->Stop();
5193}
5194
5195TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5196 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005197 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005198 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005199 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005200 fake_encoder_.SetTemporalLayersSupported(0, false);
5201
5202 // Temporal layers not supported by the encoder.
5203 // Total bitrate should be at ti:0.
5204 VideoBitrateAllocation expected_bitrate;
5205 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
5206
5207 VerifyAllocatedBitrate(expected_bitrate);
5208 video_stream_encoder_->Stop();
5209}
5210
5211TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005212 webrtc::test::ScopedFieldTrials field_trials(
5213 "WebRTC-Video-QualityScalerSettings/"
5214 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5215 // Reset encoder for field trials to take effect.
5216 ConfigureEncoder(video_encoder_config_.Copy());
5217
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005218 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005219 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005220 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005221 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005222 fake_encoder_.SetTemporalLayersSupported(0, true);
5223 fake_encoder_.SetTemporalLayersSupported(1, false);
5224
5225 const int kS0Bps = 150000;
5226 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005227 kS0Bps *
5228 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5229 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005230 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005231 kS0Bps *
5232 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5233 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005234 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
5235 // Temporal layers not supported by si:1.
5236 VideoBitrateAllocation expected_bitrate;
5237 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5238 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5239 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5240
5241 VerifyAllocatedBitrate(expected_bitrate);
5242 video_stream_encoder_->Stop();
5243}
5244
Niels Möller7dc26b72017-12-06 10:27:48 +01005245TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5246 const int kFrameWidth = 1280;
5247 const int kFrameHeight = 720;
5248 const int kFramerate = 24;
5249
Henrik Boström381d1092020-05-12 18:49:07 +02005250 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005251 DataRate::BitsPerSec(kTargetBitrateBps),
5252 DataRate::BitsPerSec(kTargetBitrateBps),
5253 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005254 test::FrameForwarder source;
5255 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005256 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005257
5258 // Insert a single frame, triggering initial configuration.
5259 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5260 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5261
5262 EXPECT_EQ(
5263 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5264 kDefaultFramerate);
5265
5266 // Trigger reconfigure encoder (without resetting the entire instance).
5267 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005268 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5269 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005270 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005271 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005272 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274
5275 // Detector should be updated with fps limit from codec config.
5276 EXPECT_EQ(
5277 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5278 kFramerate);
5279
5280 // Trigger overuse, max framerate should be reduced.
5281 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5282 stats.input_frame_rate = kFramerate;
5283 stats_proxy_->SetMockStats(stats);
5284 video_stream_encoder_->TriggerCpuOveruse();
5285 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5286 int adapted_framerate =
5287 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5288 EXPECT_LT(adapted_framerate, kFramerate);
5289
5290 // Trigger underuse, max framerate should go back to codec configured fps.
5291 // Set extra low fps, to make sure it's actually reset, not just incremented.
5292 stats = stats_proxy_->GetStats();
5293 stats.input_frame_rate = adapted_framerate / 2;
5294 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005295 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5297 EXPECT_EQ(
5298 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5299 kFramerate);
5300
5301 video_stream_encoder_->Stop();
5302}
5303
5304TEST_F(VideoStreamEncoderTest,
5305 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5306 const int kFrameWidth = 1280;
5307 const int kFrameHeight = 720;
5308 const int kLowFramerate = 15;
5309 const int kHighFramerate = 25;
5310
Henrik Boström381d1092020-05-12 18:49:07 +02005311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005312 DataRate::BitsPerSec(kTargetBitrateBps),
5313 DataRate::BitsPerSec(kTargetBitrateBps),
5314 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005315 test::FrameForwarder source;
5316 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005317 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005318
5319 // Trigger initial configuration.
5320 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005321 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5322 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005323 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005324 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005325 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005326 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005327 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5328
5329 EXPECT_EQ(
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5331 kLowFramerate);
5332
5333 // Trigger overuse, max framerate should be reduced.
5334 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5335 stats.input_frame_rate = kLowFramerate;
5336 stats_proxy_->SetMockStats(stats);
5337 video_stream_encoder_->TriggerCpuOveruse();
5338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5339 int adapted_framerate =
5340 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5341 EXPECT_LT(adapted_framerate, kLowFramerate);
5342
5343 // Reconfigure the encoder with a new (higher max framerate), max fps should
5344 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005345 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005346 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5347 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005348 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5350
5351 EXPECT_EQ(
5352 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5353 adapted_framerate);
5354
5355 // Trigger underuse, max framerate should go back to codec configured fps.
5356 stats = stats_proxy_->GetStats();
5357 stats.input_frame_rate = adapted_framerate;
5358 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005359 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005360 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5361 EXPECT_EQ(
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5363 kHighFramerate);
5364
5365 video_stream_encoder_->Stop();
5366}
5367
mflodmancc3d4422017-08-03 08:27:51 -07005368TEST_F(VideoStreamEncoderTest,
5369 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005370 const int kFrameWidth = 1280;
5371 const int kFrameHeight = 720;
5372 const int kFramerate = 24;
5373
Henrik Boström381d1092020-05-12 18:49:07 +02005374 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005375 DataRate::BitsPerSec(kTargetBitrateBps),
5376 DataRate::BitsPerSec(kTargetBitrateBps),
5377 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005378 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005379 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005380 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005381
5382 // Trigger initial configuration.
5383 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005384 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5385 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005386 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005387 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005388 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005389 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005391
Niels Möller7dc26b72017-12-06 10:27:48 +01005392 EXPECT_EQ(
5393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5394 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005395
5396 // Trigger overuse, max framerate should be reduced.
5397 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5398 stats.input_frame_rate = kFramerate;
5399 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005400 video_stream_encoder_->TriggerCpuOveruse();
5401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005402 int adapted_framerate =
5403 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005404 EXPECT_LT(adapted_framerate, kFramerate);
5405
5406 // Change degradation preference to not enable framerate scaling. Target
5407 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005408 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005409 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005410 EXPECT_EQ(
5411 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5412 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005413
mflodmancc3d4422017-08-03 08:27:51 -07005414 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005415}
5416
mflodmancc3d4422017-08-03 08:27:51 -07005417TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005418 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005419 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005420 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5421 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5422 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005423 const int kWidth = 640;
5424 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005425
asaperssonfab67072017-04-04 05:51:49 -07005426 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005427
5428 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005429 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005430
5431 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005432 EXPECT_TRUE_WAIT(
5433 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005434
sprangc5d62e22017-04-02 23:53:04 -07005435 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005436
asaperssonfab67072017-04-04 05:51:49 -07005437 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005438 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005439 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005440
5441 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005442 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005443
Henrik Boström2671dac2020-05-19 16:29:09 +02005444 EXPECT_TRUE_WAIT(
5445 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005446
mflodmancc3d4422017-08-03 08:27:51 -07005447 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005448}
5449
mflodmancc3d4422017-08-03 08:27:51 -07005450TEST_F(VideoStreamEncoderTest,
5451 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005452 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005454 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5455 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5456 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005457 const int kWidth = 640;
5458 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005459
5460 // We expect the n initial frames to get dropped.
5461 int i;
5462 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005463 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005464 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005465 }
5466 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005467 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005468 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005469
5470 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005471 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005472
mflodmancc3d4422017-08-03 08:27:51 -07005473 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005474}
5475
mflodmancc3d4422017-08-03 08:27:51 -07005476TEST_F(VideoStreamEncoderTest,
5477 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005478 const int kWidth = 640;
5479 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005481 DataRate::BitsPerSec(kLowTargetBitrateBps),
5482 DataRate::BitsPerSec(kLowTargetBitrateBps),
5483 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005484
5485 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005486 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005487 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005488
asaperssonfab67072017-04-04 05:51:49 -07005489 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005490 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005491 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005492
mflodmancc3d4422017-08-03 08:27:51 -07005493 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005494}
5495
mflodmancc3d4422017-08-03 08:27:51 -07005496TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005497 const int kWidth = 640;
5498 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005499 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005500
5501 VideoEncoderConfig video_encoder_config;
5502 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5503 // Make format different, to force recreation of encoder.
5504 video_encoder_config.video_format.parameters["foo"] = "foo";
5505 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005506 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005508 DataRate::BitsPerSec(kLowTargetBitrateBps),
5509 DataRate::BitsPerSec(kLowTargetBitrateBps),
5510 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005511
kthelgasonb83797b2017-02-14 11:57:25 -08005512 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005513 video_stream_encoder_->SetSource(&video_source_,
5514 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005515
asaperssonfab67072017-04-04 05:51:49 -07005516 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005517 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005518 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005519
mflodmancc3d4422017-08-03 08:27:51 -07005520 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005521 fake_encoder_.SetQualityScaling(true);
5522}
5523
Åsa Persson139f4dc2019-08-02 09:29:58 +02005524TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5525 webrtc::test::ScopedFieldTrials field_trials(
5526 "WebRTC-Video-QualityScalerSettings/"
5527 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5528 // Reset encoder for field trials to take effect.
5529 ConfigureEncoder(video_encoder_config_.Copy());
5530 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5531 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5532 const int kWidth = 640;
5533 const int kHeight = 360;
5534
Henrik Boström381d1092020-05-12 18:49:07 +02005535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005536 DataRate::BitsPerSec(kTargetBitrateBps),
5537 DataRate::BitsPerSec(kTargetBitrateBps),
5538 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005539 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5540 // Frame should not be dropped.
5541 WaitForEncodedFrame(1);
5542
Henrik Boström381d1092020-05-12 18:49:07 +02005543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005544 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5545 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5546 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005547 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5548 // Frame should not be dropped.
5549 WaitForEncodedFrame(2);
5550
Henrik Boström381d1092020-05-12 18:49:07 +02005551 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005552 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5553 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5554 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005555 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5556 // Expect to drop this frame, the wait should time out.
5557 ExpectDroppedFrame();
5558
5559 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005560 EXPECT_TRUE_WAIT(
5561 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005562 video_stream_encoder_->Stop();
5563}
5564
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005565TEST_F(VideoStreamEncoderTest,
5566 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5567 webrtc::test::ScopedFieldTrials field_trials(
5568 "WebRTC-Video-QualityScalerSettings/"
5569 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5570 fake_encoder_.SetQualityScaling(false);
5571 ConfigureEncoder(video_encoder_config_.Copy());
5572 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5573 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5574 const int kWidth = 640;
5575 const int kHeight = 360;
5576
5577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5578 DataRate::BitsPerSec(kTargetBitrateBps),
5579 DataRate::BitsPerSec(kTargetBitrateBps),
5580 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5581 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5582 // Frame should not be dropped.
5583 WaitForEncodedFrame(1);
5584
5585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5586 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5587 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5588 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5589 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5590 // Frame should not be dropped.
5591 WaitForEncodedFrame(2);
5592
5593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5594 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5595 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5596 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5597 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5598 // Not dropped since quality scaling is disabled.
5599 WaitForEncodedFrame(3);
5600
5601 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005602 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005603 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5604
5605 video_stream_encoder_->Stop();
5606}
5607
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005608TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5609 const int kLowTargetBitrateBps = 400000;
5610 // Set simulcast.
5611 ResetEncoder("VP8", 3, 1, 1, false);
5612 fake_encoder_.SetQualityScaling(true);
5613 const int kWidth = 1280;
5614 const int kHeight = 720;
5615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5616 DataRate::BitsPerSec(kLowTargetBitrateBps),
5617 DataRate::BitsPerSec(kLowTargetBitrateBps),
5618 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5619 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5620 // Frame should not be dropped.
5621 WaitForEncodedFrame(1);
5622
5623 // Trigger QVGA "singlecast"
5624 // Update the config.
5625 VideoEncoderConfig video_encoder_config;
5626 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5627 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005628 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005629 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005630 "VP8", /*max qp*/ 56, /*screencast*/ false,
5631 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005632 for (auto& layer : video_encoder_config.simulcast_layers) {
5633 layer.num_temporal_layers = 1;
5634 layer.max_framerate = kDefaultFramerate;
5635 }
5636 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5637 video_encoder_config.content_type =
5638 VideoEncoderConfig::ContentType::kRealtimeVideo;
5639
5640 video_encoder_config.simulcast_layers[0].active = true;
5641 video_encoder_config.simulcast_layers[1].active = false;
5642 video_encoder_config.simulcast_layers[2].active = false;
5643
5644 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5645 kMaxPayloadLength);
5646 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5647
5648 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5649 // Frame should not be dropped.
5650 WaitForEncodedFrame(2);
5651
5652 // Trigger HD "singlecast"
5653 video_encoder_config.simulcast_layers[0].active = false;
5654 video_encoder_config.simulcast_layers[1].active = false;
5655 video_encoder_config.simulcast_layers[2].active = true;
5656
5657 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5658 kMaxPayloadLength);
5659 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5660
5661 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5662 // Frame should be dropped because of initial frame drop.
5663 ExpectDroppedFrame();
5664
5665 // Expect the sink_wants to specify a scaled frame.
5666 EXPECT_TRUE_WAIT(
5667 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5668 video_stream_encoder_->Stop();
5669}
5670
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005671TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5672 const int kLowTargetBitrateBps = 400000;
5673 // Set simulcast.
5674 ResetEncoder("VP9", 1, 1, 3, false);
5675 fake_encoder_.SetQualityScaling(true);
5676 const int kWidth = 1280;
5677 const int kHeight = 720;
5678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5679 DataRate::BitsPerSec(kLowTargetBitrateBps),
5680 DataRate::BitsPerSec(kLowTargetBitrateBps),
5681 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5682 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5683 // Frame should not be dropped.
5684 WaitForEncodedFrame(1);
5685
5686 // Trigger QVGA "singlecast"
5687 // Update the config.
5688 VideoEncoderConfig video_encoder_config;
5689 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5690 &video_encoder_config);
5691 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5692 vp9_settings.numberOfSpatialLayers = 3;
5693 // Since only one layer is active - automatic resize should be enabled.
5694 vp9_settings.automaticResizeOn = true;
5695 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005696 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005697 vp9_settings);
5698 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5699 video_encoder_config.content_type =
5700 VideoEncoderConfig::ContentType::kRealtimeVideo;
5701 // Currently simulcast layers |active| flags are used to inidicate
5702 // which SVC layers are active.
5703 video_encoder_config.simulcast_layers.resize(3);
5704
5705 video_encoder_config.simulcast_layers[0].active = true;
5706 video_encoder_config.simulcast_layers[1].active = false;
5707 video_encoder_config.simulcast_layers[2].active = false;
5708
5709 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5710 kMaxPayloadLength);
5711 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5712
5713 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5714 // Frame should not be dropped.
5715 WaitForEncodedFrame(2);
5716
5717 // Trigger HD "singlecast"
5718 video_encoder_config.simulcast_layers[0].active = false;
5719 video_encoder_config.simulcast_layers[1].active = false;
5720 video_encoder_config.simulcast_layers[2].active = true;
5721
5722 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5723 kMaxPayloadLength);
5724 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5725
5726 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5727 // Frame should be dropped because of initial frame drop.
5728 ExpectDroppedFrame();
5729
5730 // Expect the sink_wants to specify a scaled frame.
5731 EXPECT_TRUE_WAIT(
5732 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5733 video_stream_encoder_->Stop();
5734}
5735
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005736TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005737 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5738 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5739 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5740 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5741 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5742 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5743 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5744 fake_encoder_.SetResolutionBitrateLimits(
5745 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5746
5747 VideoEncoderConfig video_encoder_config;
5748 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5749 &video_encoder_config);
5750 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5751 vp9_settings.numberOfSpatialLayers = 3;
5752 // Since only one layer is active - automatic resize should be enabled.
5753 vp9_settings.automaticResizeOn = true;
5754 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005755 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005756 vp9_settings);
5757 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5758 video_encoder_config.content_type =
5759 VideoEncoderConfig::ContentType::kRealtimeVideo;
5760 // Simulcast layers are used to indicate which spatial layers are active.
5761 video_encoder_config.simulcast_layers.resize(3);
5762 video_encoder_config.simulcast_layers[0].active = false;
5763 video_encoder_config.simulcast_layers[1].active = true;
5764 video_encoder_config.simulcast_layers[2].active = false;
5765
5766 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5767 kMaxPayloadLength);
5768 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5769
5770 // The encoder bitrate limits for 360p should be used.
5771 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5772 EXPECT_FALSE(WaitForFrame(1000));
5773 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5774 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5775 VideoCodecType::kVideoCodecVP9);
5776 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5777 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5778 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5779 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5780 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5781 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5782 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5783 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5784
5785 // The encoder bitrate limits for 270p should be used.
5786 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5787 EXPECT_FALSE(WaitForFrame(1000));
5788 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5789 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5790 VideoCodecType::kVideoCodecVP9);
5791 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5792 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5793 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5794 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5795 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5796 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5797 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5798 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5799
5800 video_stream_encoder_->Stop();
5801}
5802
5803TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005804 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5805 VideoEncoderConfig video_encoder_config;
5806 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5807 &video_encoder_config);
5808 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5809 vp9_settings.numberOfSpatialLayers = 3;
5810 // Since only one layer is active - automatic resize should be enabled.
5811 vp9_settings.automaticResizeOn = true;
5812 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005813 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005814 vp9_settings);
5815 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5816 video_encoder_config.content_type =
5817 VideoEncoderConfig::ContentType::kRealtimeVideo;
5818 // Simulcast layers are used to indicate which spatial layers are active.
5819 video_encoder_config.simulcast_layers.resize(3);
5820 video_encoder_config.simulcast_layers[0].active = false;
5821 video_encoder_config.simulcast_layers[1].active = true;
5822 video_encoder_config.simulcast_layers[2].active = false;
5823
5824 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5825 kMaxPayloadLength);
5826 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5827
5828 // The default bitrate limits for 360p should be used.
5829 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005830 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5831 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005832 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5833 EXPECT_FALSE(WaitForFrame(1000));
5834 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5835 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5836 VideoCodecType::kVideoCodecVP9);
5837 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5838 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5839 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5840 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5841 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5842 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5843 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5844 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5845
5846 // The default bitrate limits for 270p should be used.
5847 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005848 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5849 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005850 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5851 EXPECT_FALSE(WaitForFrame(1000));
5852 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5853 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5854 VideoCodecType::kVideoCodecVP9);
5855 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5856 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5857 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5858 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5859 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5860 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5861 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5862 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5863
5864 video_stream_encoder_->Stop();
5865}
5866
5867TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5868 webrtc::test::ScopedFieldTrials field_trials(
5869 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5870 VideoEncoderConfig video_encoder_config;
5871 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5872 &video_encoder_config);
5873 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5874 vp9_settings.numberOfSpatialLayers = 3;
5875 // Since only one layer is active - automatic resize should be enabled.
5876 vp9_settings.automaticResizeOn = true;
5877 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005878 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005879 vp9_settings);
5880 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5881 video_encoder_config.content_type =
5882 VideoEncoderConfig::ContentType::kRealtimeVideo;
5883 // Simulcast layers are used to indicate which spatial layers are active.
5884 video_encoder_config.simulcast_layers.resize(3);
5885 video_encoder_config.simulcast_layers[0].active = false;
5886 video_encoder_config.simulcast_layers[1].active = true;
5887 video_encoder_config.simulcast_layers[2].active = false;
5888
5889 // Reset encoder for field trials to take effect.
5890 ConfigureEncoder(video_encoder_config.Copy());
5891
5892 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5893 kMaxPayloadLength);
5894 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5895
5896 // The default bitrate limits for 360p should not be used.
5897 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005898 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5899 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005900 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5901 EXPECT_FALSE(WaitForFrame(1000));
5902 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005903 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005904 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5905 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5906 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5907 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5908 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5909 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5910
5911 video_stream_encoder_->Stop();
5912}
5913
5914TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5915 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5916 /*num_spatial_layers=*/1, /*screenshare=*/false);
5917
5918 // The default singlecast bitrate limits for 720p should not be used.
5919 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005920 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5921 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005922 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5923 EXPECT_FALSE(WaitForFrame(1000));
5924 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5925 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5926 VideoCodecType::kVideoCodecVP9);
5927 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5928 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5929 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5930 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5931 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5932 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5933
5934 video_stream_encoder_->Stop();
5935}
5936
5937TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005938 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5939 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5940 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5941 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5942 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5943 fake_encoder_.SetResolutionBitrateLimits(
5944 {kEncoderLimits180p, kEncoderLimits720p});
5945
5946 VideoEncoderConfig video_encoder_config;
5947 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5948 &video_encoder_config);
5949 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5950 vp9_settings.numberOfSpatialLayers = 3;
5951 // Since only one layer is active - automatic resize should be enabled.
5952 vp9_settings.automaticResizeOn = true;
5953 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005954 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005955 vp9_settings);
5956 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5957 video_encoder_config.content_type =
5958 VideoEncoderConfig::ContentType::kRealtimeVideo;
5959 // Simulcast layers are used to indicate which spatial layers are active.
5960 video_encoder_config.simulcast_layers.resize(3);
5961 video_encoder_config.simulcast_layers[0].active = true;
5962 video_encoder_config.simulcast_layers[1].active = false;
5963 video_encoder_config.simulcast_layers[2].active = false;
5964
5965 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5966 kMaxPayloadLength);
5967 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5968
5969 // Limits not applied on lowest stream, limits for 180p should not be used.
5970 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5971 EXPECT_FALSE(WaitForFrame(1000));
5972 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5973 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5974 VideoCodecType::kVideoCodecVP9);
5975 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5976 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5977 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5978 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5979 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5980 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5981 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5982 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5983
5984 video_stream_encoder_->Stop();
5985}
5986
5987TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005988 InitialFrameDropActivatesWhenResolutionIncreases) {
5989 const int kWidth = 640;
5990 const int kHeight = 360;
5991
5992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5993 DataRate::BitsPerSec(kTargetBitrateBps),
5994 DataRate::BitsPerSec(kTargetBitrateBps),
5995 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5996 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5997 // Frame should not be dropped.
5998 WaitForEncodedFrame(1);
5999
6000 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6001 DataRate::BitsPerSec(kLowTargetBitrateBps),
6002 DataRate::BitsPerSec(kLowTargetBitrateBps),
6003 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6004 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6005 // Frame should not be dropped, bitrate not too low for frame.
6006 WaitForEncodedFrame(2);
6007
6008 // Incoming resolution increases.
6009 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6010 // Expect to drop this frame, bitrate too low for frame.
6011 ExpectDroppedFrame();
6012
6013 // Expect the sink_wants to specify a scaled frame.
6014 EXPECT_TRUE_WAIT(
6015 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6016 video_stream_encoder_->Stop();
6017}
6018
6019TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6020 const int kWidth = 640;
6021 const int kHeight = 360;
6022 // So that quality scaling doesn't happen by itself.
6023 fake_encoder_.SetQp(kQpHigh);
6024
6025 AdaptingFrameForwarder source(&time_controller_);
6026 source.set_adaptation_enabled(true);
6027 video_stream_encoder_->SetSource(
6028 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6029
6030 int timestamp = 1;
6031
6032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6033 DataRate::BitsPerSec(kTargetBitrateBps),
6034 DataRate::BitsPerSec(kTargetBitrateBps),
6035 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6036 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6037 WaitForEncodedFrame(timestamp);
6038 timestamp += 9000;
6039 // Long pause to disable all first BWE drop logic.
6040 AdvanceTime(TimeDelta::Millis(1000));
6041
6042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6043 DataRate::BitsPerSec(kLowTargetBitrateBps),
6044 DataRate::BitsPerSec(kLowTargetBitrateBps),
6045 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6046 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6047 // Not dropped frame, as initial frame drop is disabled by now.
6048 WaitForEncodedFrame(timestamp);
6049 timestamp += 9000;
6050 AdvanceTime(TimeDelta::Millis(100));
6051
6052 // Quality adaptation down.
6053 video_stream_encoder_->TriggerQualityLow();
6054
6055 // Adaptation has an effect.
6056 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6057 5000);
6058
6059 // Frame isn't dropped as initial frame dropper is disabled.
6060 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6061 WaitForEncodedFrame(timestamp);
6062 timestamp += 9000;
6063 AdvanceTime(TimeDelta::Millis(100));
6064
6065 // Quality adaptation up.
6066 video_stream_encoder_->TriggerQualityHigh();
6067
6068 // Adaptation has an effect.
6069 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6070 5000);
6071
6072 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6073 // Frame should not be dropped, as initial framedropper is off.
6074 WaitForEncodedFrame(timestamp);
6075
6076 video_stream_encoder_->Stop();
6077}
6078
Åsa Persson7f354f82021-02-04 15:52:15 +01006079TEST_F(VideoStreamEncoderTest,
6080 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6081 const int kMinStartBps360p = 222000;
6082 fake_encoder_.SetResolutionBitrateLimits(
6083 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6084 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6085 800000)});
6086
6087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6088 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6089 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6091 0, 0, 0);
6092 // Frame should not be dropped, bitrate not too low for frame.
6093 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6094 WaitForEncodedFrame(1);
6095
6096 // Incoming resolution increases, initial frame drop activates.
6097 // Frame should be dropped, link allocation too low for frame.
6098 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6099 ExpectDroppedFrame();
6100
6101 // Expect sink_wants to specify a scaled frame.
6102 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6103 5000);
6104 video_stream_encoder_->Stop();
6105}
6106
6107TEST_F(VideoStreamEncoderTest,
6108 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6109 const int kMinStartBps360p = 222000;
6110 fake_encoder_.SetResolutionBitrateLimits(
6111 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6112 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6113 800000)});
6114
6115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6116 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6117 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6118 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6119 0, 0, 0);
6120 // Frame should not be dropped, bitrate not too low for frame.
6121 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6122 WaitForEncodedFrame(1);
6123
6124 // Incoming resolution increases, initial frame drop activates.
6125 // Frame should be dropped, link allocation not too low for frame.
6126 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6127 WaitForEncodedFrame(2);
6128
6129 video_stream_encoder_->Stop();
6130}
6131
Åsa Perssone644a032019-11-08 15:56:00 +01006132TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6133 webrtc::test::ScopedFieldTrials field_trials(
6134 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
6135
6136 // Reset encoder for field trials to take effect.
6137 VideoEncoderConfig config = video_encoder_config_.Copy();
6138 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02006139 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006140 ConfigureEncoder(std::move(config));
6141 fake_encoder_.SetQp(kQpLow);
6142
6143 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006144 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006145 source.set_adaptation_enabled(true);
6146 video_stream_encoder_->SetSource(&source,
6147 DegradationPreference::MAINTAIN_FRAMERATE);
6148
6149 // Start at low bitrate.
6150 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02006151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6152 DataRate::BitsPerSec(kLowBitrateBps),
6153 DataRate::BitsPerSec(kLowBitrateBps),
6154 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006155
6156 // Expect first frame to be dropped and resolution to be limited.
6157 const int kWidth = 1280;
6158 const int kHeight = 720;
6159 const int64_t kFrameIntervalMs = 100;
6160 int64_t timestamp_ms = kFrameIntervalMs;
6161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6162 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006163 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6164 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006165
6166 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006167 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6168 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006169
6170 // Insert frames and advance |min_duration_ms|.
6171 for (size_t i = 1; i <= 10; i++) {
6172 timestamp_ms += kFrameIntervalMs;
6173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6174 WaitForEncodedFrame(timestamp_ms);
6175 }
6176 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6177 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6178
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006179 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01006180
6181 // Insert frame should trigger high BW and release quality limitation.
6182 timestamp_ms += kFrameIntervalMs;
6183 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6184 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006185 // The ramp-up code involves the adaptation queue, give it time to execute.
6186 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006187 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006188 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006189
6190 // Frame should not be adapted.
6191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(kWidth, kHeight);
6194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6195
6196 video_stream_encoder_->Stop();
6197}
6198
mflodmancc3d4422017-08-03 08:27:51 -07006199TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006200 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006201 webrtc::test::ScopedFieldTrials field_trials(
6202 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006203 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006204 source.set_adaptation_enabled(true);
6205 video_stream_encoder_->SetSource(&source,
6206 DegradationPreference::MAINTAIN_FRAMERATE);
6207 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6208 DataRate::BitsPerSec(kTargetBitrateBps),
6209 DataRate::BitsPerSec(kTargetBitrateBps),
6210 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6211 fake_encoder_.SetQp(kQpHigh + 1);
6212 const int kWidth = 1280;
6213 const int kHeight = 720;
6214 const int64_t kFrameIntervalMs = 100;
6215 int64_t timestamp_ms = kFrameIntervalMs;
6216 for (size_t i = 1; i <= 100; i++) {
6217 timestamp_ms += kFrameIntervalMs;
6218 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6219 WaitForEncodedFrame(timestamp_ms);
6220 }
6221 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6222 // for the first time.
6223 // TODO(eshr): We should avoid these waits by using threads with simulated
6224 // time.
6225 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6226 2000 * 2.5 * 2);
6227 timestamp_ms += kFrameIntervalMs;
6228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6229 WaitForEncodedFrame(timestamp_ms);
6230 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6231 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6232 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6233
6234 // Disable Quality scaling by turning off scaler on the encoder and
6235 // reconfiguring.
6236 fake_encoder_.SetQualityScaling(false);
6237 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6238 kMaxPayloadLength);
6239 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006240 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006241 // Since we turned off the quality scaler, the adaptations made by it are
6242 // removed.
6243 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6245
6246 video_stream_encoder_->Stop();
6247}
6248
6249TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006250 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6251 const int kTooSmallWidth = 10;
6252 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006253 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006254 DataRate::BitsPerSec(kTargetBitrateBps),
6255 DataRate::BitsPerSec(kTargetBitrateBps),
6256 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006257
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006258 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006259 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006260 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006261 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006262 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006263 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6264
6265 // Trigger adapt down, too small frame, expect no change.
6266 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006267 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006268 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006269 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006270 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6271 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6272
mflodmancc3d4422017-08-03 08:27:51 -07006273 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006274}
6275
mflodmancc3d4422017-08-03 08:27:51 -07006276TEST_F(VideoStreamEncoderTest,
6277 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006278 const int kTooSmallWidth = 10;
6279 const int kTooSmallHeight = 10;
6280 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006281 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006282 DataRate::BitsPerSec(kTargetBitrateBps),
6283 DataRate::BitsPerSec(kTargetBitrateBps),
6284 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006285
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006286 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006287 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006288 video_stream_encoder_->SetSource(&source,
6289 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006290 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6293
6294 // Trigger adapt down, expect limited framerate.
6295 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006296 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006297 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006298 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6300 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6301 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6302
6303 // Trigger adapt down, too small frame, expect no change.
6304 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006305 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006306 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006307 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6309 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6310 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6311
mflodmancc3d4422017-08-03 08:27:51 -07006312 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006313}
6314
mflodmancc3d4422017-08-03 08:27:51 -07006315TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006316 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006318 DataRate::BitsPerSec(kTargetBitrateBps),
6319 DataRate::BitsPerSec(kTargetBitrateBps),
6320 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006321 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006322 const int kFrameWidth = 1280;
6323 const int kFrameHeight = 720;
6324 video_source_.IncomingCapturedFrame(
6325 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006326 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006327 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006328}
6329
sprangb1ca0732017-02-01 08:38:12 -08006330// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006331TEST_F(VideoStreamEncoderTest,
6332 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006334 DataRate::BitsPerSec(kTargetBitrateBps),
6335 DataRate::BitsPerSec(kTargetBitrateBps),
6336 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006337
6338 const int kFrameWidth = 1280;
6339 const int kFrameHeight = 720;
6340 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006341 // requested by
6342 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006343 video_source_.set_adaptation_enabled(true);
6344
6345 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006346 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006347 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006348
6349 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006350 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006351 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006352 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006353 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006354
asaperssonfab67072017-04-04 05:51:49 -07006355 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006356 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006357 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006358 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006359 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006360
mflodmancc3d4422017-08-03 08:27:51 -07006361 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006362}
sprangfe627f32017-03-29 08:24:59 -07006363
mflodmancc3d4422017-08-03 08:27:51 -07006364TEST_F(VideoStreamEncoderTest,
6365 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006366 const int kFrameWidth = 1280;
6367 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006368
Henrik Boström381d1092020-05-12 18:49:07 +02006369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006370 DataRate::BitsPerSec(kTargetBitrateBps),
6371 DataRate::BitsPerSec(kTargetBitrateBps),
6372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006374 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006375 video_source_.set_adaptation_enabled(true);
6376
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006377 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006378
6379 video_source_.IncomingCapturedFrame(
6380 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006381 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006382
6383 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006385
6386 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006387 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006388 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006391 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006392 }
6393
6394 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006395 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006396 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006397 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006398 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006399 video_source_.IncomingCapturedFrame(
6400 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006401 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006402 ++num_frames_dropped;
6403 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006404 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006405 }
6406 }
6407
sprang4847ae62017-06-27 07:06:52 -07006408 // Add some slack to account for frames dropped by the frame dropper.
6409 const int kErrorMargin = 1;
6410 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006411 kErrorMargin);
6412
6413 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006415 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006416 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006417 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006418 video_source_.IncomingCapturedFrame(
6419 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006420 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006421 ++num_frames_dropped;
6422 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006423 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006424 }
6425 }
sprang4847ae62017-06-27 07:06:52 -07006426 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006427 kErrorMargin);
6428
6429 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006430 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006431 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006432 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006433 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006434 video_source_.IncomingCapturedFrame(
6435 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006436 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006437 ++num_frames_dropped;
6438 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006439 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006440 }
6441 }
sprang4847ae62017-06-27 07:06:52 -07006442 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006443 kErrorMargin);
6444
6445 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006446 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006447 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006448 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006449 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006450 video_source_.IncomingCapturedFrame(
6451 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006452 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006453 ++num_frames_dropped;
6454 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006455 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006456 }
6457 }
6458 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6459
mflodmancc3d4422017-08-03 08:27:51 -07006460 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006461}
6462
mflodmancc3d4422017-08-03 08:27:51 -07006463TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006464 const int kFramerateFps = 5;
6465 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006466 const int kFrameWidth = 1280;
6467 const int kFrameHeight = 720;
6468
sprang4847ae62017-06-27 07:06:52 -07006469 // Reconfigure encoder with two temporal layers and screensharing, which will
6470 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006471 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006472
Henrik Boström381d1092020-05-12 18:49:07 +02006473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006474 DataRate::BitsPerSec(kTargetBitrateBps),
6475 DataRate::BitsPerSec(kTargetBitrateBps),
6476 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006477 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006478 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006479 video_source_.set_adaptation_enabled(true);
6480
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006481 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006482
6483 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006484 rtc::VideoSinkWants last_wants;
6485 do {
6486 last_wants = video_source_.sink_wants();
6487
sprangc5d62e22017-04-02 23:53:04 -07006488 // Insert frames to get a new fps estimate...
6489 for (int j = 0; j < kFramerateFps; ++j) {
6490 video_source_.IncomingCapturedFrame(
6491 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006492 if (video_source_.last_sent_width()) {
6493 sink_.WaitForEncodedFrame(timestamp_ms);
6494 }
sprangc5d62e22017-04-02 23:53:04 -07006495 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006496 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006497 }
6498 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006499 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006500 } while (video_source_.sink_wants().max_framerate_fps <
6501 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006502
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006503 EXPECT_THAT(video_source_.sink_wants(),
6504 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006505
mflodmancc3d4422017-08-03 08:27:51 -07006506 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006507}
asaperssonf7e294d2017-06-13 23:25:22 -07006508
mflodmancc3d4422017-08-03 08:27:51 -07006509TEST_F(VideoStreamEncoderTest,
6510 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006511 const int kWidth = 1280;
6512 const int kHeight = 720;
6513 const int64_t kFrameIntervalMs = 150;
6514 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006516 DataRate::BitsPerSec(kTargetBitrateBps),
6517 DataRate::BitsPerSec(kTargetBitrateBps),
6518 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006519
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006520 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006521 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006522 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006523 video_stream_encoder_->SetSource(&source,
6524 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006525 timestamp_ms += kFrameIntervalMs;
6526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006527 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6532
6533 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006534 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006538 EXPECT_THAT(source.sink_wants(),
6539 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6542 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6543
6544 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006545 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006546 timestamp_ms += kFrameIntervalMs;
6547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006549 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6551 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6552 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6553
6554 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006555 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006556 timestamp_ms += kFrameIntervalMs;
6557 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006558 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006559 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6562 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6563
6564 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006565 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006566 timestamp_ms += kFrameIntervalMs;
6567 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006568 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006569 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6572 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6573
6574 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006575 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006576 timestamp_ms += kFrameIntervalMs;
6577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006578 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006579 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6582 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6583
6584 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006585 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006586 timestamp_ms += kFrameIntervalMs;
6587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006588 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006589 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6592 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6593
6594 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006595 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006596 timestamp_ms += kFrameIntervalMs;
6597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006598 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006599 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006600 rtc::VideoSinkWants last_wants = source.sink_wants();
6601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6603 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6604
6605 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006606 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006607 timestamp_ms += kFrameIntervalMs;
6608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006609 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006610 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6613 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6614
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006615 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006616 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006617 timestamp_ms += kFrameIntervalMs;
6618 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006619 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006620 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6623 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6624
6625 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006626 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006627 timestamp_ms += kFrameIntervalMs;
6628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006629 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006630 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6633 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6634
6635 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006636 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006637 timestamp_ms += kFrameIntervalMs;
6638 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006639 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006640 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6643 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6644
6645 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006646 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006647 timestamp_ms += kFrameIntervalMs;
6648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006649 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006650 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6652 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6653 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6654
6655 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006656 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006657 timestamp_ms += kFrameIntervalMs;
6658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006659 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006660 EXPECT_THAT(source.sink_wants(), FpsMax());
6661 EXPECT_EQ(source.sink_wants().max_pixel_count,
6662 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006663 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6665 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6666
6667 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006668 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006669 timestamp_ms += kFrameIntervalMs;
6670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006671 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006672 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006673 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6675 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6676
Åsa Persson30ab0152019-08-27 12:22:33 +02006677 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006678 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006679 timestamp_ms += kFrameIntervalMs;
6680 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006681 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006682 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006684 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6686 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6687
6688 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006689 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006690 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006691 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6692
mflodmancc3d4422017-08-03 08:27:51 -07006693 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006694}
6695
mflodmancc3d4422017-08-03 08:27:51 -07006696TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006697 const int kWidth = 1280;
6698 const int kHeight = 720;
6699 const int64_t kFrameIntervalMs = 150;
6700 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006701 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006702 DataRate::BitsPerSec(kTargetBitrateBps),
6703 DataRate::BitsPerSec(kTargetBitrateBps),
6704 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006705
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006706 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006707 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006708 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006709 video_stream_encoder_->SetSource(&source,
6710 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006711 timestamp_ms += kFrameIntervalMs;
6712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006713 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6717 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6718 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6719 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6720 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6721
6722 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006723 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006724 timestamp_ms += kFrameIntervalMs;
6725 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006726 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006727 EXPECT_THAT(source.sink_wants(),
6728 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6731 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6732 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6733 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6735
6736 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006737 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006738 timestamp_ms += kFrameIntervalMs;
6739 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006740 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006741 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6744 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6745 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6746 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6748
6749 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006750 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006751 timestamp_ms += kFrameIntervalMs;
6752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006753 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006754 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6757 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6758 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6759 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6760 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6761
Evan Shrubsole64469032020-06-11 10:45:29 +02006762 // Trigger cpu adapt up, expect no change since QP is most limited.
6763 {
6764 // Store current sink wants since we expect no change and if there is no
6765 // change then last_wants() is not updated.
6766 auto previous_sink_wants = source.sink_wants();
6767 video_stream_encoder_->TriggerCpuUnderuse();
6768 timestamp_ms += kFrameIntervalMs;
6769 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6770 WaitForEncodedFrame(timestamp_ms);
6771 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6772 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6773 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6774 }
6775
6776 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6777 video_stream_encoder_->TriggerQualityHigh();
6778 timestamp_ms += kFrameIntervalMs;
6779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6780 WaitForEncodedFrame(timestamp_ms);
6781 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6782 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6784 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6786 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6787 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6788
6789 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6790 // expect increased resolution (960x540@30fps).
6791 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006792 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006793 timestamp_ms += kFrameIntervalMs;
6794 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006795 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006796 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006797 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6799 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6800 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6801 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006802 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006803
Evan Shrubsole64469032020-06-11 10:45:29 +02006804 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6805 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006806 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006807 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006808 timestamp_ms += kFrameIntervalMs;
6809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006810 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006811 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006812 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006813 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6814 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6815 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6816 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6817 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006818 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006819
6820 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006821 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006822 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006823 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006824 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006825
mflodmancc3d4422017-08-03 08:27:51 -07006826 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006827}
6828
mflodmancc3d4422017-08-03 08:27:51 -07006829TEST_F(VideoStreamEncoderTest,
6830 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006831 const int kWidth = 640;
6832 const int kHeight = 360;
6833 const int kFpsLimit = 15;
6834 const int64_t kFrameIntervalMs = 150;
6835 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006837 DataRate::BitsPerSec(kTargetBitrateBps),
6838 DataRate::BitsPerSec(kTargetBitrateBps),
6839 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006840
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006841 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006842 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006843 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006844 video_stream_encoder_->SetSource(&source,
6845 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006846 timestamp_ms += kFrameIntervalMs;
6847 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006848 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006849 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006850 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6852 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6854 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6855 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6856
6857 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006858 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006859 timestamp_ms += kFrameIntervalMs;
6860 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006861 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006862 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6866 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6867 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6868 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6869
6870 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006871 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006872 timestamp_ms += kFrameIntervalMs;
6873 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006874 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006875 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006876 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006878 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6879 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6880 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6881 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6882
Evan Shrubsole64469032020-06-11 10:45:29 +02006883 // Trigger cpu adapt up, expect no change because quality is most limited.
6884 {
6885 auto previous_sink_wants = source.sink_wants();
6886 // Store current sink wants since we expect no change ind if there is no
6887 // change then last__wants() is not updated.
6888 video_stream_encoder_->TriggerCpuUnderuse();
6889 timestamp_ms += kFrameIntervalMs;
6890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6891 WaitForEncodedFrame(timestamp_ms);
6892 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6893 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6894 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6895 }
6896
6897 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6898 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006899 timestamp_ms += kFrameIntervalMs;
6900 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006901 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006902 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6904 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6905 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006906 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6907 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6908 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006909
Evan Shrubsole64469032020-06-11 10:45:29 +02006910 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006911 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006912 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006913 timestamp_ms += kFrameIntervalMs;
6914 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006915 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006916 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006917 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6918 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6919 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6921 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006922 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006923
6924 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006925 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006926 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006927 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006928 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006929
mflodmancc3d4422017-08-03 08:27:51 -07006930 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006931}
6932
mflodmancc3d4422017-08-03 08:27:51 -07006933TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006934 const int kFrameWidth = 1920;
6935 const int kFrameHeight = 1080;
6936 // 3/4 of 1920.
6937 const int kAdaptedFrameWidth = 1440;
6938 // 3/4 of 1080 rounded down to multiple of 4.
6939 const int kAdaptedFrameHeight = 808;
6940 const int kFramerate = 24;
6941
Henrik Boström381d1092020-05-12 18:49:07 +02006942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006943 DataRate::BitsPerSec(kTargetBitrateBps),
6944 DataRate::BitsPerSec(kTargetBitrateBps),
6945 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006946 // Trigger reconfigure encoder (without resetting the entire instance).
6947 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006948 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6949 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006950 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006951 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006952 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006953 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006954 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006955 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006956
6957 video_source_.set_adaptation_enabled(true);
6958
6959 video_source_.IncomingCapturedFrame(
6960 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006961 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006962
6963 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006964 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006965 video_source_.IncomingCapturedFrame(
6966 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006967 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006968
mflodmancc3d4422017-08-03 08:27:51 -07006969 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006970}
6971
mflodmancc3d4422017-08-03 08:27:51 -07006972TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006973 const int kFrameWidth = 1280;
6974 const int kFrameHeight = 720;
6975 const int kLowFps = 2;
6976 const int kHighFps = 30;
6977
Henrik Boström381d1092020-05-12 18:49:07 +02006978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006979 DataRate::BitsPerSec(kTargetBitrateBps),
6980 DataRate::BitsPerSec(kTargetBitrateBps),
6981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006982
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006983 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006984 max_framerate_ = kLowFps;
6985
6986 // Insert 2 seconds of 2fps video.
6987 for (int i = 0; i < kLowFps * 2; ++i) {
6988 video_source_.IncomingCapturedFrame(
6989 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6990 WaitForEncodedFrame(timestamp_ms);
6991 timestamp_ms += 1000 / kLowFps;
6992 }
6993
6994 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006996 DataRate::BitsPerSec(kTargetBitrateBps),
6997 DataRate::BitsPerSec(kTargetBitrateBps),
6998 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006999 video_source_.IncomingCapturedFrame(
7000 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7001 WaitForEncodedFrame(timestamp_ms);
7002 timestamp_ms += 1000 / kLowFps;
7003
7004 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
7005
7006 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02007007 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07007008 const int kFrameIntervalMs = 1000 / kHighFps;
7009 max_framerate_ = kHighFps;
7010 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
7011 video_source_.IncomingCapturedFrame(
7012 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7013 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7014 // be dropped if the encoder hans't been updated with the new higher target
7015 // framerate yet, causing it to overshoot the target bitrate and then
7016 // suffering the wrath of the media optimizer.
7017 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7018 timestamp_ms += kFrameIntervalMs;
7019 }
7020
7021 // Don expect correct measurement just yet, but it should be higher than
7022 // before.
7023 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7024
mflodmancc3d4422017-08-03 08:27:51 -07007025 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007026}
7027
mflodmancc3d4422017-08-03 08:27:51 -07007028TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007029 const int kFrameWidth = 1280;
7030 const int kFrameHeight = 720;
7031 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007032 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007033 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007034 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007035
Henrik Boström381d1092020-05-12 18:49:07 +02007036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007037 DataRate::BitsPerSec(kTargetBitrateBps),
7038 DataRate::BitsPerSec(kTargetBitrateBps),
7039 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007040 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007041
7042 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007043 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007044 video_source_.IncomingCapturedFrame(
7045 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7046 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007047 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007048
7049 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7051 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
7052 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007053
7054 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007055 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007056 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007057
Per Kjellanderdcef6412020-10-07 15:09:05 +02007058 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007059 video_source_.IncomingCapturedFrame(
7060 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7061 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007062 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007063
mflodmancc3d4422017-08-03 08:27:51 -07007064 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007065}
ilnik6b826ef2017-06-16 06:53:48 -07007066
Niels Möller4db138e2018-04-19 09:04:13 +02007067TEST_F(VideoStreamEncoderTest,
7068 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7069 const int kFrameWidth = 1280;
7070 const int kFrameHeight = 720;
7071 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007073 DataRate::BitsPerSec(kTargetBitrateBps),
7074 DataRate::BitsPerSec(kTargetBitrateBps),
7075 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007076 video_source_.IncomingCapturedFrame(
7077 CreateFrame(1, kFrameWidth, kFrameHeight));
7078 WaitForEncodedFrame(1);
7079 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7080 .low_encode_usage_threshold_percent,
7081 default_options.low_encode_usage_threshold_percent);
7082 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7083 .high_encode_usage_threshold_percent,
7084 default_options.high_encode_usage_threshold_percent);
7085 video_stream_encoder_->Stop();
7086}
7087
7088TEST_F(VideoStreamEncoderTest,
7089 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7090 const int kFrameWidth = 1280;
7091 const int kFrameHeight = 720;
7092 CpuOveruseOptions hardware_options;
7093 hardware_options.low_encode_usage_threshold_percent = 150;
7094 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007095 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007096
Henrik Boström381d1092020-05-12 18:49:07 +02007097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007098 DataRate::BitsPerSec(kTargetBitrateBps),
7099 DataRate::BitsPerSec(kTargetBitrateBps),
7100 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007101 video_source_.IncomingCapturedFrame(
7102 CreateFrame(1, kFrameWidth, kFrameHeight));
7103 WaitForEncodedFrame(1);
7104 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7105 .low_encode_usage_threshold_percent,
7106 hardware_options.low_encode_usage_threshold_percent);
7107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7108 .high_encode_usage_threshold_percent,
7109 hardware_options.high_encode_usage_threshold_percent);
7110 video_stream_encoder_->Stop();
7111}
7112
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007113TEST_F(VideoStreamEncoderTest,
7114 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7115 const int kFrameWidth = 1280;
7116 const int kFrameHeight = 720;
7117
7118 const CpuOveruseOptions default_options;
7119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7120 DataRate::BitsPerSec(kTargetBitrateBps),
7121 DataRate::BitsPerSec(kTargetBitrateBps),
7122 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7123 video_source_.IncomingCapturedFrame(
7124 CreateFrame(1, kFrameWidth, kFrameHeight));
7125 WaitForEncodedFrame(1);
7126 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7127 .low_encode_usage_threshold_percent,
7128 default_options.low_encode_usage_threshold_percent);
7129 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7130 .high_encode_usage_threshold_percent,
7131 default_options.high_encode_usage_threshold_percent);
7132
7133 CpuOveruseOptions hardware_options;
7134 hardware_options.low_encode_usage_threshold_percent = 150;
7135 hardware_options.high_encode_usage_threshold_percent = 200;
7136 fake_encoder_.SetIsHardwareAccelerated(true);
7137
7138 video_source_.IncomingCapturedFrame(
7139 CreateFrame(2, kFrameWidth, kFrameHeight));
7140 WaitForEncodedFrame(2);
7141
7142 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7143 .low_encode_usage_threshold_percent,
7144 hardware_options.low_encode_usage_threshold_percent);
7145 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7146 .high_encode_usage_threshold_percent,
7147 hardware_options.high_encode_usage_threshold_percent);
7148
7149 video_stream_encoder_->Stop();
7150}
7151
Niels Möller6bb5ab92019-01-11 11:11:10 +01007152TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7153 const int kFrameWidth = 320;
7154 const int kFrameHeight = 240;
7155 const int kFps = 30;
7156 const int kTargetBitrateBps = 120000;
7157 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7158
Henrik Boström381d1092020-05-12 18:49:07 +02007159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007160 DataRate::BitsPerSec(kTargetBitrateBps),
7161 DataRate::BitsPerSec(kTargetBitrateBps),
7162 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007163
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007164 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007165 max_framerate_ = kFps;
7166
7167 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7168 fake_encoder_.SimulateOvershoot(1.0);
7169 int num_dropped = 0;
7170 for (int i = 0; i < kNumFramesInRun; ++i) {
7171 video_source_.IncomingCapturedFrame(
7172 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7173 // Wait up to two frame durations for a frame to arrive.
7174 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7175 ++num_dropped;
7176 }
7177 timestamp_ms += 1000 / kFps;
7178 }
7179
Erik Språnga8d48ab2019-02-08 14:17:40 +01007180 // Framerate should be measured to be near the expected target rate.
7181 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7182
7183 // Frame drops should be within 5% of expected 0%.
7184 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007185
7186 // Make encoder produce frames at double the expected bitrate during 3 seconds
7187 // of video, verify number of drops. Rate needs to be slightly changed in
7188 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007189 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007190 const RateControlSettings trials =
7191 RateControlSettings::ParseFromFieldTrials();
7192 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007193 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007194 // frame dropping since the adjuter will try to just lower the target
7195 // bitrate rather than drop frames. If network headroom can be used, it
7196 // doesn't push back as hard so we don't need quite as much overshoot.
7197 // These numbers are unfortunately a bit magical but there's not trivial
7198 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007199 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007200 }
7201 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007202 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007203 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7204 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7205 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007206 num_dropped = 0;
7207 for (int i = 0; i < kNumFramesInRun; ++i) {
7208 video_source_.IncomingCapturedFrame(
7209 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7210 // Wait up to two frame durations for a frame to arrive.
7211 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7212 ++num_dropped;
7213 }
7214 timestamp_ms += 1000 / kFps;
7215 }
7216
Henrik Boström381d1092020-05-12 18:49:07 +02007217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007218 DataRate::BitsPerSec(kTargetBitrateBps),
7219 DataRate::BitsPerSec(kTargetBitrateBps),
7220 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007221
7222 // Target framerate should be still be near the expected target, despite
7223 // the frame drops.
7224 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7225
7226 // Frame drops should be within 5% of expected 50%.
7227 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007228
7229 video_stream_encoder_->Stop();
7230}
7231
7232TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7233 const int kFrameWidth = 320;
7234 const int kFrameHeight = 240;
7235 const int kActualInputFps = 24;
7236 const int kTargetBitrateBps = 120000;
7237
7238 ASSERT_GT(max_framerate_, kActualInputFps);
7239
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007240 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007241 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007243 DataRate::BitsPerSec(kTargetBitrateBps),
7244 DataRate::BitsPerSec(kTargetBitrateBps),
7245 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007246
7247 // Insert 3 seconds of video, with an input fps lower than configured max.
7248 for (int i = 0; i < kActualInputFps * 3; ++i) {
7249 video_source_.IncomingCapturedFrame(
7250 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7251 // Wait up to two frame durations for a frame to arrive.
7252 WaitForEncodedFrame(timestamp_ms);
7253 timestamp_ms += 1000 / kActualInputFps;
7254 }
7255
7256 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7257
7258 video_stream_encoder_->Stop();
7259}
7260
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007261TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007262 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007264 DataRate::BitsPerSec(kTargetBitrateBps),
7265 DataRate::BitsPerSec(kTargetBitrateBps),
7266 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007267
7268 fake_encoder_.BlockNextEncode();
7269 video_source_.IncomingCapturedFrame(
7270 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7271 WaitForEncodedFrame(1);
7272 // On the very first frame full update should be forced.
7273 rect = fake_encoder_.GetLastUpdateRect();
7274 EXPECT_EQ(rect.offset_x, 0);
7275 EXPECT_EQ(rect.offset_y, 0);
7276 EXPECT_EQ(rect.height, codec_height_);
7277 EXPECT_EQ(rect.width, codec_width_);
7278 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7279 // call to ContinueEncode.
7280 video_source_.IncomingCapturedFrame(
7281 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7282 ExpectDroppedFrame();
7283 video_source_.IncomingCapturedFrame(
7284 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7285 ExpectDroppedFrame();
7286 fake_encoder_.ContinueEncode();
7287 WaitForEncodedFrame(3);
7288 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7289 rect = fake_encoder_.GetLastUpdateRect();
7290 EXPECT_EQ(rect.offset_x, 1);
7291 EXPECT_EQ(rect.offset_y, 0);
7292 EXPECT_EQ(rect.width, 10);
7293 EXPECT_EQ(rect.height, 1);
7294
7295 video_source_.IncomingCapturedFrame(
7296 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7297 WaitForEncodedFrame(4);
7298 // Previous frame was encoded, so no accumulation should happen.
7299 rect = fake_encoder_.GetLastUpdateRect();
7300 EXPECT_EQ(rect.offset_x, 0);
7301 EXPECT_EQ(rect.offset_y, 0);
7302 EXPECT_EQ(rect.width, 1);
7303 EXPECT_EQ(rect.height, 1);
7304
7305 video_stream_encoder_->Stop();
7306}
7307
Erik Språngd7329ca2019-02-21 21:19:53 +01007308TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007309 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007310 DataRate::BitsPerSec(kTargetBitrateBps),
7311 DataRate::BitsPerSec(kTargetBitrateBps),
7312 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007313
7314 // First frame is always keyframe.
7315 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7316 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007317 EXPECT_THAT(
7318 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007319 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007320
7321 // Insert delta frame.
7322 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7323 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007324 EXPECT_THAT(
7325 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007326 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007327
7328 // Request next frame be a key-frame.
7329 video_stream_encoder_->SendKeyFrame();
7330 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7331 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007332 EXPECT_THAT(
7333 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007334 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007335
7336 video_stream_encoder_->Stop();
7337}
7338
7339TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7340 // Setup simulcast with three streams.
7341 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007343 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7344 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7345 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007346 // Wait for all three layers before triggering event.
7347 sink_.SetNumExpectedLayers(3);
7348
7349 // First frame is always keyframe.
7350 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7351 WaitForEncodedFrame(1);
7352 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007353 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7354 VideoFrameType::kVideoFrameKey,
7355 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007356
7357 // Insert delta frame.
7358 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7359 WaitForEncodedFrame(2);
7360 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007361 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7362 VideoFrameType::kVideoFrameDelta,
7363 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007364
7365 // Request next frame be a key-frame.
7366 // Only first stream is configured to produce key-frame.
7367 video_stream_encoder_->SendKeyFrame();
7368 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7369 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007370
7371 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7372 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007373 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007374 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007375 VideoFrameType::kVideoFrameKey,
7376 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007377
7378 video_stream_encoder_->Stop();
7379}
7380
7381TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7382 // Configure internal source factory and setup test again.
7383 encoder_factory_.SetHasInternalSource(true);
7384 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007386 DataRate::BitsPerSec(kTargetBitrateBps),
7387 DataRate::BitsPerSec(kTargetBitrateBps),
7388 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007389
7390 // Call encoder directly, simulating internal source where encoded frame
7391 // callback in VideoStreamEncoder is called despite no OnFrame().
7392 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7393 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007394 EXPECT_THAT(
7395 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007396 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007397
Niels Möller8f7ce222019-03-21 15:43:58 +01007398 const std::vector<VideoFrameType> kDeltaFrame = {
7399 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007400 // Need to set timestamp manually since manually for injected frame.
7401 VideoFrame frame = CreateFrame(101, nullptr);
7402 frame.set_timestamp(101);
7403 fake_encoder_.InjectFrame(frame, false);
7404 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007405 EXPECT_THAT(
7406 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007407 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007408
7409 // Request key-frame. The forces a dummy frame down into the encoder.
7410 fake_encoder_.ExpectNullFrame();
7411 video_stream_encoder_->SendKeyFrame();
7412 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007413 EXPECT_THAT(
7414 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007415 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007416
7417 video_stream_encoder_->Stop();
7418}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007419
7420TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7421 // Configure internal source factory and setup test again.
7422 encoder_factory_.SetHasInternalSource(true);
7423 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007425 DataRate::BitsPerSec(kTargetBitrateBps),
7426 DataRate::BitsPerSec(kTargetBitrateBps),
7427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007428
7429 int64_t timestamp = 1;
7430 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007431 image.capture_time_ms_ = ++timestamp;
7432 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7433 const int64_t kEncodeFinishDelayMs = 10;
7434 image.timing_.encode_start_ms = timestamp;
7435 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007436 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007437 // Wait for frame without incrementing clock.
7438 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7439 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7440 // capture timestamp should be kEncodeFinishDelayMs in the past.
7441 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007442 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007443
7444 video_stream_encoder_->Stop();
7445}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007446
7447TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007448 // SPS contains VUI with restrictions on the maximum number of reordered
7449 // pictures, there is no need to rewrite the bitstream to enable faster
7450 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007451 ResetEncoder("H264", 1, 1, 1, false);
7452
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7454 DataRate::BitsPerSec(kTargetBitrateBps),
7455 DataRate::BitsPerSec(kTargetBitrateBps),
7456 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7457 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007458
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007459 fake_encoder_.SetEncodedImageData(
7460 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007461
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007462 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7463 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007464
7465 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7466 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007467
7468 video_stream_encoder_->Stop();
7469}
7470
7471TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007472 // SPS does not contain VUI, the bitstream is will be rewritten with added
7473 // VUI with restrictions on the maximum number of reordered pictures to
7474 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007475 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7476 0x00, 0x00, 0x03, 0x03, 0xF4,
7477 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007478 ResetEncoder("H264", 1, 1, 1, false);
7479
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7481 DataRate::BitsPerSec(kTargetBitrateBps),
7482 DataRate::BitsPerSec(kTargetBitrateBps),
7483 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7484 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007485
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007486 fake_encoder_.SetEncodedImageData(
7487 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007488
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007489 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7490 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007491
7492 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7493 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007494
7495 video_stream_encoder_->Stop();
7496}
7497
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007498TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7499 const int kFrameWidth = 1280;
7500 const int kFrameHeight = 720;
7501 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7502
Henrik Boström381d1092020-05-12 18:49:07 +02007503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007504 DataRate::BitsPerSec(kTargetBitrateBps),
7505 DataRate::BitsPerSec(kTargetBitrateBps),
7506 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007507 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7508
7509 // Insert a first video frame. It should be dropped because of downscale in
7510 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007511 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007512 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7513 frame.set_rotation(kVideoRotation_270);
7514 video_source_.IncomingCapturedFrame(frame);
7515
7516 ExpectDroppedFrame();
7517
7518 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007519 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007520 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7521 frame.set_rotation(kVideoRotation_90);
7522 video_source_.IncomingCapturedFrame(frame);
7523
7524 WaitForEncodedFrame(timestamp_ms);
7525 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7526
7527 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007528 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007529 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7530 frame.set_rotation(kVideoRotation_180);
7531 video_source_.IncomingCapturedFrame(frame);
7532
7533 WaitForEncodedFrame(timestamp_ms);
7534 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7535
7536 video_stream_encoder_->Stop();
7537}
7538
Erik Språng5056af02019-09-02 15:53:11 +02007539TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7540 const int kFrameWidth = 320;
7541 const int kFrameHeight = 180;
7542
7543 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007545 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7546 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7547 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007548 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007549 /*rtt_ms=*/0,
7550 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007551
7552 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007553 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007554 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7555 frame.set_rotation(kVideoRotation_270);
7556 video_source_.IncomingCapturedFrame(frame);
7557 WaitForEncodedFrame(timestamp_ms);
7558
7559 // Set a target rate below the minimum allowed by the codec settings.
7560 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007561 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7562 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007564 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007565 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007566 /*link_allocation=*/target_rate,
7567 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007568 /*rtt_ms=*/0,
7569 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007570 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7571
7572 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7573 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7574 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007575 DataRate allocation_sum =
7576 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007577 EXPECT_EQ(min_rate, allocation_sum);
7578 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7579
7580 video_stream_encoder_->Stop();
7581}
7582
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007583TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007585 DataRate::BitsPerSec(kTargetBitrateBps),
7586 DataRate::BitsPerSec(kTargetBitrateBps),
7587 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007588 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007589 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007590 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7591 WaitForEncodedFrame(1);
7592
7593 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7594 ASSERT_TRUE(prev_rate_settings.has_value());
7595 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7596 kDefaultFramerate);
7597
7598 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7599 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7600 timestamp_ms += 1000 / kDefaultFramerate;
7601 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7602 WaitForEncodedFrame(timestamp_ms);
7603 }
7604 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7605 kDefaultFramerate);
7606 // Capture larger frame to trigger a reconfigure.
7607 codec_height_ *= 2;
7608 codec_width_ *= 2;
7609 timestamp_ms += 1000 / kDefaultFramerate;
7610 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7611 WaitForEncodedFrame(timestamp_ms);
7612
7613 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7614 auto current_rate_settings =
7615 fake_encoder_.GetAndResetLastRateControlSettings();
7616 // Ensure we have actually reconfigured twice
7617 // The rate settings should have been set again even though
7618 // they haven't changed.
7619 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007620 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007621
7622 video_stream_encoder_->Stop();
7623}
7624
philipeld9cc8c02019-09-16 14:53:40 +02007625struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007626 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7627 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7628 MOCK_METHOD(void,
7629 RequestEncoderSwitch,
7630 (const webrtc::SdpVideoFormat& format),
7631 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007632};
7633
philipel9b058032020-02-10 11:30:00 +01007634TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7635 constexpr int kDontCare = 100;
7636 StrictMock<MockEncoderSelector> encoder_selector;
7637 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7638 &fake_encoder_, &encoder_selector);
7639 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7640
7641 // Reset encoder for new configuration to take effect.
7642 ConfigureEncoder(video_encoder_config_.Copy());
7643
7644 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7645
7646 video_source_.IncomingCapturedFrame(
7647 CreateFrame(kDontCare, kDontCare, kDontCare));
7648 video_stream_encoder_->Stop();
7649
7650 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7651 // to it's factory, so in order for the encoder instance in the
7652 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7653 // reset the |video_stream_encoder_| here.
7654 video_stream_encoder_.reset();
7655}
7656
7657TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7658 constexpr int kDontCare = 100;
7659
7660 NiceMock<MockEncoderSelector> encoder_selector;
7661 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7662 video_send_config_.encoder_settings.encoder_switch_request_callback =
7663 &switch_callback;
7664 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7665 &fake_encoder_, &encoder_selector);
7666 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7667
7668 // Reset encoder for new configuration to take effect.
7669 ConfigureEncoder(video_encoder_config_.Copy());
7670
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007671 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007672 .WillByDefault(Return(SdpVideoFormat("AV1")));
7673 EXPECT_CALL(switch_callback,
7674 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7675 Field(&SdpVideoFormat::name, "AV1"))));
7676
Henrik Boström381d1092020-05-12 18:49:07 +02007677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007678 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7679 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7680 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007681 /*fraction_lost=*/0,
7682 /*rtt_ms=*/0,
7683 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007684 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007685
7686 video_stream_encoder_->Stop();
7687}
7688
7689TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7690 constexpr int kSufficientBitrateToNotDrop = 1000;
7691 constexpr int kDontCare = 100;
7692
7693 NiceMock<MockVideoEncoder> video_encoder;
7694 NiceMock<MockEncoderSelector> encoder_selector;
7695 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7696 video_send_config_.encoder_settings.encoder_switch_request_callback =
7697 &switch_callback;
7698 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7699 &video_encoder, &encoder_selector);
7700 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7701
7702 // Reset encoder for new configuration to take effect.
7703 ConfigureEncoder(video_encoder_config_.Copy());
7704
7705 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7706 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7707 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007709 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7710 /*stable_target_bitrate=*/
7711 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7712 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007713 /*fraction_lost=*/0,
7714 /*rtt_ms=*/0,
7715 /*cwnd_reduce_ratio=*/0);
7716
7717 ON_CALL(video_encoder, Encode(_, _))
7718 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7719 ON_CALL(encoder_selector, OnEncoderBroken())
7720 .WillByDefault(Return(SdpVideoFormat("AV2")));
7721
7722 rtc::Event encode_attempted;
7723 EXPECT_CALL(switch_callback,
7724 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7725 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7726 EXPECT_EQ(format.name, "AV2");
7727 encode_attempted.Set();
7728 });
7729
7730 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7731 encode_attempted.Wait(3000);
7732
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007733 AdvanceTime(TimeDelta::Millis(0));
7734
philipel9b058032020-02-10 11:30:00 +01007735 video_stream_encoder_->Stop();
7736
7737 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7738 // to it's factory, so in order for the encoder instance in the
7739 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7740 // reset the |video_stream_encoder_| here.
7741 video_stream_encoder_.reset();
7742}
7743
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007744TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007745 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007746 const int kFrameWidth = 320;
7747 const int kFrameHeight = 180;
7748
7749 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007750 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007752 /*target_bitrate=*/rate,
7753 /*stable_target_bitrate=*/rate,
7754 /*link_allocation=*/rate,
7755 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007756 /*rtt_ms=*/0,
7757 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007758
7759 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007760 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007761 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7762 frame.set_rotation(kVideoRotation_270);
7763 video_source_.IncomingCapturedFrame(frame);
7764 WaitForEncodedFrame(timestamp_ms);
7765 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7766
7767 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007768 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007769 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007770 /*target_bitrate=*/new_stable_rate,
7771 /*stable_target_bitrate=*/new_stable_rate,
7772 /*link_allocation=*/rate,
7773 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007774 /*rtt_ms=*/0,
7775 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7777 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7778 video_stream_encoder_->Stop();
7779}
7780
7781TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007782 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007783 const int kFrameWidth = 320;
7784 const int kFrameHeight = 180;
7785
7786 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007787 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007789 /*target_bitrate=*/rate,
7790 /*stable_target_bitrate=*/rate,
7791 /*link_allocation=*/rate,
7792 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007793 /*rtt_ms=*/0,
7794 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007795
7796 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007797 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007798 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7799 frame.set_rotation(kVideoRotation_270);
7800 video_source_.IncomingCapturedFrame(frame);
7801 WaitForEncodedFrame(timestamp_ms);
7802 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7803
7804 // Set a higher target rate without changing the link_allocation. Should not
7805 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007806 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007808 /*target_bitrate=*/rate,
7809 /*stable_target_bitrate=*/new_stable_rate,
7810 /*link_allocation=*/rate,
7811 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007812 /*rtt_ms=*/0,
7813 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7815 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7816 video_stream_encoder_->Stop();
7817}
7818
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007819TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7820 test::ScopedFieldTrials field_trials(
7821 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7822 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7823 const int kFramerateFps = 30;
7824 const int kWidth = 1920;
7825 const int kHeight = 1080;
7826 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7827 // Works on screenshare mode.
7828 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7829 // We rely on the automatic resolution adaptation, but we handle framerate
7830 // adaptation manually by mocking the stats proxy.
7831 video_source_.set_adaptation_enabled(true);
7832
7833 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007835 DataRate::BitsPerSec(kTargetBitrateBps),
7836 DataRate::BitsPerSec(kTargetBitrateBps),
7837 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007838 video_stream_encoder_->SetSource(&video_source_,
7839 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007840 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007841
7842 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7843 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7844
7845 // Pass enough frames with the full update to trigger animation detection.
7846 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007847 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007848 frame.set_ntp_time_ms(timestamp_ms);
7849 frame.set_timestamp_us(timestamp_ms * 1000);
7850 video_source_.IncomingCapturedFrame(frame);
7851 WaitForEncodedFrame(timestamp_ms);
7852 }
7853
7854 // Resolution should be limited.
7855 rtc::VideoSinkWants expected;
7856 expected.max_framerate_fps = kFramerateFps;
7857 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007858 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007859
7860 // Pass one frame with no known update.
7861 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007862 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007863 frame.set_ntp_time_ms(timestamp_ms);
7864 frame.set_timestamp_us(timestamp_ms * 1000);
7865 frame.clear_update_rect();
7866
7867 video_source_.IncomingCapturedFrame(frame);
7868 WaitForEncodedFrame(timestamp_ms);
7869
7870 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007871 EXPECT_THAT(video_source_.sink_wants(),
7872 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007873
7874 video_stream_encoder_->Stop();
7875}
7876
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007877TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7878 const int kWidth = 720; // 540p adapted down.
7879 const int kHeight = 405;
7880 const int kNumFrames = 3;
7881 // Works on screenshare mode.
7882 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7883 /*num_spatial_layers=*/2, /*screenshare=*/true);
7884
7885 video_source_.set_adaptation_enabled(true);
7886
7887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7888 DataRate::BitsPerSec(kTargetBitrateBps),
7889 DataRate::BitsPerSec(kTargetBitrateBps),
7890 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7891
7892 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7893
7894 // Pass enough frames with the full update to trigger animation detection.
7895 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007896 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007897 frame.set_ntp_time_ms(timestamp_ms);
7898 frame.set_timestamp_us(timestamp_ms * 1000);
7899 video_source_.IncomingCapturedFrame(frame);
7900 WaitForEncodedFrame(timestamp_ms);
7901 }
7902
7903 video_stream_encoder_->Stop();
7904}
7905
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007906TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7907 const float downscale_factors[] = {4.0, 2.0, 1.0};
7908 const int number_layers =
7909 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7910 VideoEncoderConfig config;
7911 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7912 for (int i = 0; i < number_layers; ++i) {
7913 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7914 config.simulcast_layers[i].active = true;
7915 }
7916 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007917 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007918 "VP8", /*max qp*/ 56, /*screencast*/ false,
7919 /*screenshare enabled*/ false);
7920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7921 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7922 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7923 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7924
7925 // First initialization.
7926 // Encoder should be initialized. Next frame should be key frame.
7927 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7928 sink_.SetNumExpectedLayers(number_layers);
7929 int64_t timestamp_ms = kFrameIntervalMs;
7930 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7931 WaitForEncodedFrame(timestamp_ms);
7932 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7933 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7934 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7935 VideoFrameType::kVideoFrameKey,
7936 VideoFrameType::kVideoFrameKey}));
7937
7938 // Disable top layer.
7939 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7940 config.simulcast_layers[number_layers - 1].active = false;
7941 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7942 sink_.SetNumExpectedLayers(number_layers - 1);
7943 timestamp_ms += kFrameIntervalMs;
7944 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7945 WaitForEncodedFrame(timestamp_ms);
7946 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7947 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7948 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7949 VideoFrameType::kVideoFrameDelta,
7950 VideoFrameType::kVideoFrameDelta}));
7951
7952 // Re-enable top layer.
7953 // Encoder should be re-initialized. Next frame should be key frame.
7954 config.simulcast_layers[number_layers - 1].active = true;
7955 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7956 sink_.SetNumExpectedLayers(number_layers);
7957 timestamp_ms += kFrameIntervalMs;
7958 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7959 WaitForEncodedFrame(timestamp_ms);
7960 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7961 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7962 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7963 VideoFrameType::kVideoFrameKey,
7964 VideoFrameType::kVideoFrameKey}));
7965
7966 // Top layer max rate change.
7967 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7968 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7969 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7970 sink_.SetNumExpectedLayers(number_layers);
7971 timestamp_ms += kFrameIntervalMs;
7972 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7973 WaitForEncodedFrame(timestamp_ms);
7974 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7975 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7976 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7977 VideoFrameType::kVideoFrameDelta,
7978 VideoFrameType::kVideoFrameDelta}));
7979
7980 // Top layer resolution change.
7981 // Encoder should be re-initialized. Next frame should be key frame.
7982 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7983 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7984 sink_.SetNumExpectedLayers(number_layers);
7985 timestamp_ms += kFrameIntervalMs;
7986 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7987 WaitForEncodedFrame(timestamp_ms);
7988 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7989 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7990 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7991 VideoFrameType::kVideoFrameKey,
7992 VideoFrameType::kVideoFrameKey}));
7993 video_stream_encoder_->Stop();
7994}
7995
Henrik Boström1124ed12021-02-25 10:30:39 +01007996TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7997 const int kFrameWidth = 1280;
7998 const int kFrameHeight = 720;
7999
8000 SetUp();
8001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8002 DataRate::BitsPerSec(kTargetBitrateBps),
8003 DataRate::BitsPerSec(kTargetBitrateBps),
8004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8005
8006 // Capturing a frame should reconfigure the encoder and expose the encoder
8007 // resolution, which is the same as the input frame.
8008 int64_t timestamp_ms = kFrameIntervalMs;
8009 video_source_.IncomingCapturedFrame(
8010 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8011 WaitForEncodedFrame(timestamp_ms);
8012 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8013 EXPECT_THAT(video_source_.sink_wants().resolutions,
8014 ::testing::ElementsAreArray(
8015 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8016
8017 video_stream_encoder_->Stop();
8018}
8019
8020TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8021 // Pick downscale factors such that we never encode at full resolution - this
8022 // is an interesting use case. The frame resolution influences the encoder
8023 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
8024 // encoder should not ask for the frame resolution. This allows video frames
8025 // to have the appearence of one resolution but optimize its internal buffers
8026 // for what is actually encoded.
8027 const size_t kNumSimulcastLayers = 3u;
8028 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8029 const int kFrameWidth = 1280;
8030 const int kFrameHeight = 720;
8031 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8032 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8033 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8034 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8035 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8036 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8037
8038 VideoEncoderConfig config;
8039 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8040 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8041 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8042 config.simulcast_layers[i].active = true;
8043 }
8044 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008045 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008046 "VP8", /*max qp*/ 56, /*screencast*/ false,
8047 /*screenshare enabled*/ false);
8048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8049 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8050 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8051 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8052
8053 // Capture a frame with all layers active.
8054 int64_t timestamp_ms = kFrameIntervalMs;
8055 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8056 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8057 video_source_.IncomingCapturedFrame(
8058 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8059 WaitForEncodedFrame(timestamp_ms);
8060 // Expect encoded resolutions to match the expected simulcast layers.
8061 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8062 EXPECT_THAT(
8063 video_source_.sink_wants().resolutions,
8064 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8065
8066 // Capture a frame with one of the layers inactive.
8067 timestamp_ms += kFrameIntervalMs;
8068 config.simulcast_layers[2].active = false;
8069 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8070 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8071 video_source_.IncomingCapturedFrame(
8072 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8073 WaitForEncodedFrame(timestamp_ms);
8074
8075 // Expect encoded resolutions to match the expected simulcast layers.
8076 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8077 EXPECT_THAT(video_source_.sink_wants().resolutions,
8078 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8079
8080 // Capture a frame with all but one layer turned off.
8081 timestamp_ms += kFrameIntervalMs;
8082 config.simulcast_layers[1].active = false;
8083 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8084 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8085 video_source_.IncomingCapturedFrame(
8086 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8087 WaitForEncodedFrame(timestamp_ms);
8088
8089 // Expect encoded resolutions to match the expected simulcast layers.
8090 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8091 EXPECT_THAT(video_source_.sink_wants().resolutions,
8092 ::testing::ElementsAreArray({kLayer0Size}));
8093
8094 video_stream_encoder_->Stop();
8095}
8096
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008097TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008098 ResetEncoder("VP8", 1, 1, 1, false);
8099
Niels Möller8b692902021-06-14 12:04:57 +02008100 // Force encoder reconfig.
8101 video_source_.IncomingCapturedFrame(
8102 CreateFrame(1, codec_width_, codec_height_));
8103 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8104
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008105 // Set QP on encoded frame and pass the frame to encode complete callback.
8106 // Since QP is present QP parsing won't be triggered and the original value
8107 // should be kept.
8108 EncodedImage encoded_image;
8109 encoded_image.qp_ = 123;
8110 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8111 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8112 CodecSpecificInfo codec_info;
8113 codec_info.codecType = kVideoCodecVP8;
8114 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8115 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8116 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8117 video_stream_encoder_->Stop();
8118}
8119
8120TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008121 ResetEncoder("VP8", 1, 1, 1, false);
8122
Niels Möller8b692902021-06-14 12:04:57 +02008123 // Force encoder reconfig.
8124 video_source_.IncomingCapturedFrame(
8125 CreateFrame(1, codec_width_, codec_height_));
8126 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8127
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008128 // Pass an encoded frame without QP to encode complete callback. QP should be
8129 // parsed and set.
8130 EncodedImage encoded_image;
8131 encoded_image.qp_ = -1;
8132 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8133 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8134 CodecSpecificInfo codec_info;
8135 codec_info.codecType = kVideoCodecVP8;
8136 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8137 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8138 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8139 video_stream_encoder_->Stop();
8140}
8141
8142TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
8143 webrtc::test::ScopedFieldTrials field_trials(
8144 "WebRTC-QpParsingKillSwitch/Enabled/");
8145
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008146 ResetEncoder("VP8", 1, 1, 1, false);
8147
Niels Möller8b692902021-06-14 12:04:57 +02008148 // Force encoder reconfig.
8149 video_source_.IncomingCapturedFrame(
8150 CreateFrame(1, codec_width_, codec_height_));
8151 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8152
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008153 EncodedImage encoded_image;
8154 encoded_image.qp_ = -1;
8155 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8156 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8157 CodecSpecificInfo codec_info;
8158 codec_info.codecType = kVideoCodecVP8;
8159 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8160 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8161 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8162 video_stream_encoder_->Stop();
8163}
8164
Sergey Silkind19e3b92021-03-16 10:05:30 +00008165TEST_F(VideoStreamEncoderTest,
8166 QualityScalingNotAllowed_QualityScalingDisabled) {
8167 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8168
8169 // Disable scaling settings in encoder info.
8170 fake_encoder_.SetQualityScaling(false);
8171 // Disable quality scaling in encoder config.
8172 video_encoder_config.is_quality_scaling_allowed = false;
8173 ConfigureEncoder(std::move(video_encoder_config));
8174
8175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8176 DataRate::BitsPerSec(kTargetBitrateBps),
8177 DataRate::BitsPerSec(kTargetBitrateBps),
8178 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8179
8180 test::FrameForwarder source;
8181 video_stream_encoder_->SetSource(
8182 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8183 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8184 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8185
8186 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8187 WaitForEncodedFrame(1);
8188 video_stream_encoder_->TriggerQualityLow();
8189 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8190
8191 video_stream_encoder_->Stop();
8192}
8193
8194#if !defined(WEBRTC_IOS)
8195// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8196// disabled by default on iOS.
8197TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8198 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8199
8200 // Disable scaling settings in encoder info.
8201 fake_encoder_.SetQualityScaling(false);
8202 // Enable quality scaling in encoder config.
8203 video_encoder_config.is_quality_scaling_allowed = true;
8204 ConfigureEncoder(std::move(video_encoder_config));
8205
8206 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8207 DataRate::BitsPerSec(kTargetBitrateBps),
8208 DataRate::BitsPerSec(kTargetBitrateBps),
8209 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8210
8211 test::FrameForwarder source;
8212 video_stream_encoder_->SetSource(
8213 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8214 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8215 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8216
8217 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8218 WaitForEncodedFrame(1);
8219 video_stream_encoder_->TriggerQualityLow();
8220 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8221
8222 video_stream_encoder_->Stop();
8223}
8224#endif
8225
Henrik Boström56db9ff2021-03-24 09:06:45 +01008226// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8227class VideoStreamEncoderWithRealEncoderTest
8228 : public VideoStreamEncoderTest,
8229 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8230 public:
8231 VideoStreamEncoderWithRealEncoderTest()
8232 : VideoStreamEncoderTest(),
8233 codec_type_(std::get<0>(GetParam())),
8234 allow_i420_conversion_(std::get<1>(GetParam())) {}
8235
8236 void SetUp() override {
8237 VideoStreamEncoderTest::SetUp();
8238 std::unique_ptr<VideoEncoder> encoder;
8239 switch (codec_type_) {
8240 case kVideoCodecVP8:
8241 encoder = VP8Encoder::Create();
8242 break;
8243 case kVideoCodecVP9:
8244 encoder = VP9Encoder::Create();
8245 break;
8246 case kVideoCodecAV1:
8247 encoder = CreateLibaomAv1Encoder();
8248 break;
8249 case kVideoCodecH264:
8250 encoder =
8251 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8252 break;
8253 case kVideoCodecMultiplex:
8254 mock_encoder_factory_for_multiplex_ =
8255 std::make_unique<MockVideoEncoderFactory>();
8256 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8257 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8258 .WillRepeatedly([] { return VP8Encoder::Create(); });
8259 encoder = std::make_unique<MultiplexEncoderAdapter>(
8260 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8261 false);
8262 break;
8263 default:
8264 RTC_NOTREACHED();
8265 }
8266 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8267 }
8268
8269 void TearDown() override {
8270 video_stream_encoder_->Stop();
8271 // Ensure |video_stream_encoder_| is destroyed before
8272 // |encoder_proxy_factory_|.
8273 video_stream_encoder_.reset();
8274 VideoStreamEncoderTest::TearDown();
8275 }
8276
8277 protected:
8278 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8279 std::unique_ptr<VideoEncoder> encoder) {
8280 // Configure VSE to use the encoder.
8281 encoder_ = std::move(encoder);
8282 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8283 encoder_.get(), &encoder_selector_);
8284 video_send_config_.encoder_settings.encoder_factory =
8285 encoder_proxy_factory_.get();
8286 VideoEncoderConfig video_encoder_config;
8287 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8288 video_encoder_config_ = video_encoder_config.Copy();
8289 ConfigureEncoder(video_encoder_config_.Copy());
8290
8291 // Set bitrate to ensure frame is not dropped.
8292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8293 DataRate::BitsPerSec(kTargetBitrateBps),
8294 DataRate::BitsPerSec(kTargetBitrateBps),
8295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8296 }
8297
8298 const VideoCodecType codec_type_;
8299 const bool allow_i420_conversion_;
8300 NiceMock<MockEncoderSelector> encoder_selector_;
8301 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8302 std::unique_ptr<VideoEncoder> encoder_;
8303 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8304};
8305
8306TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8307 auto native_i420_frame = test::CreateMappableNativeFrame(
8308 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8309 video_source_.IncomingCapturedFrame(native_i420_frame);
8310 WaitForEncodedFrame(codec_width_, codec_height_);
8311
8312 auto mappable_native_buffer =
8313 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8314 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8315 mappable_native_buffer->GetMappedFramedBuffers();
8316 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8317 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8318 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8319 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8320}
8321
8322TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8323 auto native_nv12_frame = test::CreateMappableNativeFrame(
8324 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8325 video_source_.IncomingCapturedFrame(native_nv12_frame);
8326 WaitForEncodedFrame(codec_width_, codec_height_);
8327
8328 auto mappable_native_buffer =
8329 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8330 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8331 mappable_native_buffer->GetMappedFramedBuffers();
8332 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8333 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8334 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8335 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8336
8337 if (!allow_i420_conversion_) {
8338 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8339 }
8340}
8341
Erik Språng7444b192021-06-02 14:02:13 +02008342TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8343 if (codec_type_ == kVideoCodecMultiplex) {
8344 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8345 return;
8346 }
8347
8348 const size_t kNumSpatialLayers = 3u;
8349 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8350 const int kFrameWidth = 1280;
8351 const int kFrameHeight = 720;
8352 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8353 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8354 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8355 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8356 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8357 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8358
8359 VideoEncoderConfig config;
8360 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8361 test::FillEncoderConfiguration(codec_type_, 1, &config);
8362 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8363 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8364 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8365 vp9_settings.numberOfTemporalLayers = 3;
8366 vp9_settings.automaticResizeOn = false;
8367 config.encoder_specific_settings =
8368 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8369 vp9_settings);
8370 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8371 /*fps=*/30.0,
8372 /*first_active_layer=*/0,
8373 /*num_spatial_layers=*/3,
8374 /*num_temporal_layers=*/3,
8375 /*is_screenshare=*/false);
8376 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8377 test::FillEncoderConfiguration(codec_type_, 1, &config);
8378 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8379 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8380 /*fps=*/30.0,
8381 /*first_active_layer=*/0,
8382 /*num_spatial_layers=*/3,
8383 /*num_temporal_layers=*/3,
8384 /*is_screenshare=*/false);
8385 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8386 } else {
8387 // Simulcast for VP8/H264.
8388 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8389 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8390 config.simulcast_layers[i].scale_resolution_down_by =
8391 kDownscaleFactors[i];
8392 config.simulcast_layers[i].active = true;
8393 }
8394 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8395 // Turn off frame dropping to prevent flakiness.
8396 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8397 h264_settings.frameDroppingOn = false;
8398 config.encoder_specific_settings = rtc::make_ref_counted<
8399 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8400 }
8401 }
8402
8403 auto set_layer_active = [&](int layer_idx, bool active) {
8404 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8405 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8406 config.spatial_layers[layer_idx].active = active;
8407 } else {
8408 config.simulcast_layers[layer_idx].active = active;
8409 }
8410 };
8411
8412 config.video_stream_factory =
8413 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8414 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8415 /*screencast*/ false,
8416 /*screenshare enabled*/ false);
8417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8418 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8419 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8420 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8421
8422 // Capture a frame with all layers active.
8423 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8424 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8425 int64_t timestamp_ms = kFrameIntervalMs;
8426 video_source_.IncomingCapturedFrame(
8427 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8428
8429 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8431
8432 // Capture a frame with one of the layers inactive.
8433 set_layer_active(2, false);
8434 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8435 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8436 timestamp_ms += kFrameIntervalMs;
8437 video_source_.IncomingCapturedFrame(
8438 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8439 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8440
8441 // New target bitrates signaled based on lower resolution.
8442 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8444 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8445 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8446
8447 // Re-enable the top layer.
8448 set_layer_active(2, true);
8449 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8450 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8451 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8452
8453 // Bitrate target adjusted back up to enable HD layer...
8454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8455 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8456 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8457 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8458
8459 // ...then add a new frame.
8460 timestamp_ms += kFrameIntervalMs;
8461 video_source_.IncomingCapturedFrame(
8462 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8463 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8465
8466 video_stream_encoder_->Stop();
8467}
8468
Henrik Boström56db9ff2021-03-24 09:06:45 +01008469std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8470 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8471 VideoCodecType codec_type = std::get<0>(info.param);
8472 bool allow_i420_conversion = std::get<1>(info.param);
8473 std::string str;
8474 switch (codec_type) {
8475 case kVideoCodecGeneric:
8476 str = "Generic";
8477 break;
8478 case kVideoCodecVP8:
8479 str = "VP8";
8480 break;
8481 case kVideoCodecVP9:
8482 str = "VP9";
8483 break;
8484 case kVideoCodecAV1:
8485 str = "AV1";
8486 break;
8487 case kVideoCodecH264:
8488 str = "H264";
8489 break;
8490 case kVideoCodecMultiplex:
8491 str = "Multiplex";
8492 break;
8493 default:
8494 RTC_NOTREACHED();
8495 }
8496 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8497 return str;
8498}
8499
8500constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8501 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8502constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8503 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8504constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8505 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8506constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8507 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8508#if defined(WEBRTC_USE_H264)
8509constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8510 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8511
8512// The windows compiler does not tolerate #if statements inside the
8513// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8514// and without H264).
8515INSTANTIATE_TEST_SUITE_P(
8516 All,
8517 VideoStreamEncoderWithRealEncoderTest,
8518 ::testing::Values(kVP8DisallowConversion,
8519 kVP9DisallowConversion,
8520 kAV1AllowConversion,
8521 kMultiplexDisallowConversion,
8522 kH264AllowConversion),
8523 TestParametersVideoCodecAndAllowI420ConversionToString);
8524#else
8525INSTANTIATE_TEST_SUITE_P(
8526 All,
8527 VideoStreamEncoderWithRealEncoderTest,
8528 ::testing::Values(kVP8DisallowConversion,
8529 kVP9DisallowConversion,
8530 kAV1AllowConversion,
8531 kMultiplexDisallowConversion),
8532 TestParametersVideoCodecAndAllowI420ConversionToString);
8533#endif
8534
perkj26091b12016-09-01 01:17:40 -07008535} // namespace webrtc