blob: 956610130f641391ac80af5c691ecdd5bc531869 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020045#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020046#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010047#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020048#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010049#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020050#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020053#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010054#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010062#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020063#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020064#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070066
67namespace webrtc {
68
sprang57c2fff2017-01-16 06:24:02 -080069using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020071using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020072using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010081using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080083
perkj803d97f2016-11-01 11:45:46 -070084namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080091const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010092const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020093const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010094const uint32_t kSimulcastTargetBitrateBps = 3150000;
95const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080096const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070097const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020099const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800104
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200105uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 11:45:46 -0700115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 09:06:45 +0100129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150 }
Noah Richards51db4212019-06-12 06:59:12 -0700151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 09:15:13 +0200163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 10:27:48 +0100194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 09:04:13 +0200215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 18:49:07 +0200217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219 private:
Markus Handella3765182020-07-08 13:13:32 +0200220 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100223};
224
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200227 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 08:27:51 -0700343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700344 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100358 task_queue_factory,
359 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200360 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200361 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200362 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200363 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200364 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200365 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200366 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200367 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100368 }
perkj803d97f2016-11-01 11:45:46 -0700369
Henrik Boström381d1092020-05-12 18:49:07 +0200370 void SetSourceAndWaitForRestrictionsUpdated(
371 rtc::VideoSourceInterface<VideoFrame>* source,
372 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 FakeVideoSourceRestrictionsListener listener;
374 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200375 SetSource(source, degradation_preference);
376 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200377 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200378 }
379
380 void SetSourceAndWaitForFramerateUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
383 overuse_detector_proxy_->framerate_updated_event()->Reset();
384 SetSource(source, degradation_preference);
385 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
386 }
387
388 void OnBitrateUpdatedAndWaitForManagedResources(
389 DataRate target_bitrate,
390 DataRate stable_target_bitrate,
391 DataRate link_allocation,
392 uint8_t fraction_lost,
393 int64_t round_trip_time_ms,
394 double cwnd_reduce_ratio) {
395 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
396 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
397 // Bitrate is updated on the encoder queue.
398 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200399 }
400
kthelgason2fc52542017-03-03 00:24:41 -0800401 // This is used as a synchronisation mechanism, to make sure that the
402 // encoder queue is not blocked before we start sending it frames.
403 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100404 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200405 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800406 ASSERT_TRUE(event.Wait(5000));
407 }
408
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200419
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 void TriggerCpuUnderuse() {
421 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200422 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200423 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 event.Set();
425 });
426 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200427 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200428 }
kthelgason876222f2016-11-29 01:44:11 -0800429
Henrik Boström91aa7322020-04-28 12:24:33 +0200430 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200431 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200433 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200434 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200435 event.Set();
436 });
437 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200438 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200440 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200443 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 }
449
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100451 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200452 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
453 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700455};
456
Noah Richards51db4212019-06-12 06:59:12 -0700457// Simulates simulcast behavior and makes highest stream resolutions divisible
458// by 4.
459class CroppingVideoStreamFactory
460 : public VideoEncoderConfig::VideoStreamFactoryInterface {
461 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200462 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700463
464 private:
465 std::vector<VideoStream> CreateEncoderStreams(
466 int width,
467 int height,
468 const VideoEncoderConfig& encoder_config) override {
469 std::vector<VideoStream> streams = test::CreateVideoStreams(
470 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700471 return streams;
472 }
Noah Richards51db4212019-06-12 06:59:12 -0700473};
474
sprangb1ca0732017-02-01 08:38:12 -0800475class AdaptingFrameForwarder : public test::FrameForwarder {
476 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 explicit AdaptingFrameForwarder(TimeController* time_controller)
478 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700479 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800480
481 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200482 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800483 adaptation_enabled_ = enabled;
484 }
485
asaperssonfab67072017-04-04 05:51:49 -0700486 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800488 return adaptation_enabled_;
489 }
490
Henrik Boström1124ed12021-02-25 10:30:39 +0100491 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
492 // the resolution or frame rate was different than it is currently. If
493 // something else is modified, such as encoder resolutions, but the resolution
494 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700495 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200496 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700497 return last_wants_;
498 }
499
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200500 absl::optional<int> last_sent_width() const { return last_width_; }
501 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800502
sprangb1ca0732017-02-01 08:38:12 -0800503 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200504 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
505 time_controller_->AdvanceTime(TimeDelta::Millis(0));
506
sprangb1ca0732017-02-01 08:38:12 -0800507 int cropped_width = 0;
508 int cropped_height = 0;
509 int out_width = 0;
510 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700511 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
513 << "w=" << video_frame.width()
514 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700515 if (adapter_.AdaptFrameResolution(
516 video_frame.width(), video_frame.height(),
517 video_frame.timestamp_us() * 1000, &cropped_width,
518 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame adapted_frame =
520 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200521 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100522 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200523 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-02 23:53:04 -0700534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700540 }
sprangb1ca0732017-02-01 08:38:12 -0800541 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800546 }
547 }
548
549 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
550 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200551 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100552 rtc::VideoSinkWants prev_wants = sink_wants_locked();
553 bool did_adapt =
554 prev_wants.max_pixel_count != wants.max_pixel_count ||
555 prev_wants.target_pixel_count != wants.target_pixel_count ||
556 prev_wants.max_framerate_fps != wants.max_framerate_fps;
557 if (did_adapt) {
558 last_wants_ = prev_wants;
559 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100560 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200561 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800562 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200563
564 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800565 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200566 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
567 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200568 absl::optional<int> last_width_;
569 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800570};
sprangc5d62e22017-04-02 23:53:04 -0700571
Niels Möller213618e2018-07-24 09:29:58 +0200572// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700573class MockableSendStatisticsProxy : public SendStatisticsProxy {
574 public:
575 MockableSendStatisticsProxy(Clock* clock,
576 const VideoSendStream::Config& config,
577 VideoEncoderConfig::ContentType content_type)
578 : SendStatisticsProxy(clock, config, content_type) {}
579
580 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200581 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700582 if (mock_stats_)
583 return *mock_stats_;
584 return SendStatisticsProxy::GetStats();
585 }
586
Niels Möller213618e2018-07-24 09:29:58 +0200587 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200588 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200589 if (mock_stats_)
590 return mock_stats_->input_frame_rate;
591 return SendStatisticsProxy::GetInputFrameRate();
592 }
sprangc5d62e22017-04-02 23:53:04 -0700593 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200594 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700595 mock_stats_.emplace(stats);
596 }
597
598 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200599 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700600 mock_stats_.reset();
601 }
602
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200603 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
604 on_frame_dropped_ = std::move(callback);
605 }
606
sprangc5d62e22017-04-02 23:53:04 -0700607 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200608 void OnFrameDropped(DropReason reason) override {
609 SendStatisticsProxy::OnFrameDropped(reason);
610 if (on_frame_dropped_)
611 on_frame_dropped_(reason);
612 }
613
Markus Handella3765182020-07-08 13:13:32 +0200614 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200615 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200616 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700617};
618
philipel9b058032020-02-10 11:30:00 +0100619class MockEncoderSelector
620 : public VideoEncoderFactory::EncoderSelectorInterface {
621 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200622 MOCK_METHOD(void,
623 OnCurrentEncoder,
624 (const SdpVideoFormat& format),
625 (override));
626 MOCK_METHOD(absl::optional<SdpVideoFormat>,
627 OnAvailableBitrate,
628 (const DataRate& rate),
629 (override));
630 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100631};
632
perkj803d97f2016-11-01 11:45:46 -0700633} // namespace
634
mflodmancc3d4422017-08-03 08:27:51 -0700635class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700636 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200637 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700638
mflodmancc3d4422017-08-03 08:27:51 -0700639 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700640 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700641 codec_width_(320),
642 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200643 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200644 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200645 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700646 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200647 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700648 video_send_config_,
649 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700651
652 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700653 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700654 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200655 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800656 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200657 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200658 video_send_config_.rtp.payload_name = "FAKE";
659 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700660
Per512ecb32016-09-23 15:52:06 +0200661 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200662 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200663 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
664 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
665 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100666 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700667
Niels Möllerf1338562018-04-26 09:51:47 +0200668 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800669 }
670
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100671 void ConfigureEncoder(
672 VideoEncoderConfig video_encoder_config,
673 VideoStreamEncoder::BitrateAllocationCallbackType
674 allocation_callback_type =
675 VideoStreamEncoder::BitrateAllocationCallbackType::
676 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700677 if (video_stream_encoder_)
678 video_stream_encoder_->Stop();
679 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200680 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100681 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700682 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
683 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700684 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700685 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
686 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200687 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700688 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800689 }
690
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100691 void ResetEncoder(const std::string& payload_name,
692 size_t num_streams,
693 size_t num_temporal_layers,
694 unsigned char num_spatial_layers,
695 bool screenshare,
696 VideoStreamEncoder::BitrateAllocationCallbackType
697 allocation_callback_type =
698 VideoStreamEncoder::BitrateAllocationCallbackType::
699 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200700 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800701
702 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200703 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
704 num_streams, &video_encoder_config);
705 for (auto& layer : video_encoder_config.simulcast_layers) {
706 layer.num_temporal_layers = num_temporal_layers;
707 layer.max_framerate = kDefaultFramerate;
708 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100709 video_encoder_config.max_bitrate_bps =
710 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700711 video_encoder_config.content_type =
712 screenshare ? VideoEncoderConfig::ContentType::kScreen
713 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700714 if (payload_name == "VP9") {
715 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
716 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200717 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700718 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200719 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
720 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700721 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100722 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700723 }
724
sprang57c2fff2017-01-16 06:24:02 -0800725 VideoFrame CreateFrame(int64_t ntp_time_ms,
726 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200727 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200728 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200729 destruction_event, codec_width_, codec_height_))
730 .set_ntp_time_ms(ntp_time_ms)
731 .set_timestamp_ms(99)
732 .set_rotation(kVideoRotation_0)
733 .build();
perkj26091b12016-09-01 01:17:40 -0700734 }
735
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100736 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
737 rtc::Event* destruction_event,
738 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200739 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200740 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200741 destruction_event, codec_width_, codec_height_))
742 .set_ntp_time_ms(ntp_time_ms)
743 .set_timestamp_ms(99)
744 .set_rotation(kVideoRotation_0)
745 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
746 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100747 }
748
sprang57c2fff2017-01-16 06:24:02 -0800749 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200750 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
751 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200752 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200753 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200754 .set_ntp_time_ms(ntp_time_ms)
755 .set_timestamp_ms(ntp_time_ms)
756 .set_rotation(kVideoRotation_0)
757 .build();
perkj803d97f2016-11-01 11:45:46 -0700758 }
759
Evan Shrubsole895556e2020-10-05 09:15:13 +0200760 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200761 return VideoFrame::Builder()
762 .set_video_frame_buffer(NV12Buffer::Create(width, height))
763 .set_ntp_time_ms(ntp_time_ms)
764 .set_timestamp_ms(ntp_time_ms)
765 .set_rotation(kVideoRotation_0)
766 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200767 }
768
Noah Richards51db4212019-06-12 06:59:12 -0700769 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
770 rtc::Event* destruction_event,
771 int width,
772 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200773 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200774 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200775 destruction_event, width, height))
776 .set_ntp_time_ms(ntp_time_ms)
777 .set_timestamp_ms(99)
778 .set_rotation(kVideoRotation_0)
779 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700780 }
781
Evan Shrubsole895556e2020-10-05 09:15:13 +0200782 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
783 rtc::Event* destruction_event,
784 int width,
785 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200786 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200787 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200788 destruction_event, width, height))
789 .set_ntp_time_ms(ntp_time_ms)
790 .set_timestamp_ms(99)
791 .set_rotation(kVideoRotation_0)
792 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200793 }
794
Noah Richards51db4212019-06-12 06:59:12 -0700795 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
796 rtc::Event* destruction_event) const {
797 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
798 codec_height_);
799 }
800
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100801 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100803 DataRate::BitsPerSec(kTargetBitrateBps),
804 DataRate::BitsPerSec(kTargetBitrateBps),
805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100806
807 video_source_.IncomingCapturedFrame(
808 CreateFrame(1, codec_width_, codec_height_));
809 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200810 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100811 }
812
sprang4847ae62017-06-27 07:06:52 -0700813 void WaitForEncodedFrame(int64_t expected_ntp_time) {
814 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200815 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700816 }
817
818 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
819 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200820 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700821 return ok;
822 }
823
824 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
825 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200826 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700827 }
828
829 void ExpectDroppedFrame() {
830 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200831 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700832 }
833
834 bool WaitForFrame(int64_t timeout_ms) {
835 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200836 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700837 return ok;
838 }
839
perkj26091b12016-09-01 01:17:40 -0700840 class TestEncoder : public test::FakeEncoder {
841 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200842 explicit TestEncoder(TimeController* time_controller)
843 : FakeEncoder(time_controller->GetClock()),
844 time_controller_(time_controller) {
845 RTC_DCHECK(time_controller_);
846 }
perkj26091b12016-09-01 01:17:40 -0700847
asaperssonfab67072017-04-04 05:51:49 -0700848 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200849 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700850 return config_;
851 }
852
853 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200854 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700855 block_next_encode_ = true;
856 }
857
Erik Språngaed30702018-11-05 12:57:17 +0100858 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200859 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200860 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100861 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100862 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100863 info.scaling_settings = VideoEncoder::ScalingSettings(
864 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100865 }
866 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100867 for (int i = 0; i < kMaxSpatialLayers; ++i) {
868 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100869 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100870 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100871 for (int tid = 0; tid < num_layers; ++tid)
872 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100873 }
874 }
Erik Språngaed30702018-11-05 12:57:17 +0100875 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200876
877 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100878 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200879 info.apply_alignment_to_all_simulcast_layers =
880 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200881 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100882 return info;
kthelgason876222f2016-11-29 01:44:11 -0800883 }
884
Erik Språngb7cb7b52019-02-26 15:52:33 +0100885 int32_t RegisterEncodeCompleteCallback(
886 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200887 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100888 encoded_image_callback_ = callback;
889 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
890 }
891
perkjfa10b552016-10-02 23:45:26 -0700892 void ContinueEncode() { continue_encode_event_.Set(); }
893
894 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
895 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200896 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700897 EXPECT_EQ(timestamp_, timestamp);
898 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
899 }
900
kthelgason2fc52542017-03-03 00:24:41 -0800901 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200902 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800903 quality_scaling_ = b;
904 }
kthelgasonad9010c2017-02-14 00:46:51 -0800905
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100906 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200907 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100908 requested_resolution_alignment_ = requested_resolution_alignment;
909 }
910
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200911 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
912 MutexLock lock(&local_mutex_);
913 apply_alignment_to_all_simulcast_layers_ = b;
914 }
915
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100916 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200917 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100918 is_hardware_accelerated_ = is_hardware_accelerated;
919 }
920
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100921 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
922 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200923 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100924 temporal_layers_supported_[spatial_idx] = supported;
925 }
926
Sergey Silkin6456e352019-07-08 17:56:40 +0200927 void SetResolutionBitrateLimits(
928 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200929 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200930 resolution_bitrate_limits_ = thresholds;
931 }
932
sprangfe627f32017-03-29 08:24:59 -0700933 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200934 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700935 force_init_encode_failed_ = force_failure;
936 }
937
Niels Möller6bb5ab92019-01-11 11:11:10 +0100938 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200939 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100940 rate_factor_ = rate_factor;
941 }
942
Erik Språngd7329ca2019-02-21 21:19:53 +0100943 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200944 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100945 return last_framerate_;
946 }
947
Erik Språngd7329ca2019-02-21 21:19:53 +0100948 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200949 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100950 return last_update_rect_;
951 }
952
Niels Möller87e2d782019-03-07 10:18:23 +0100953 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200954 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 return last_frame_types_;
956 }
957
958 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100959 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100960 keyframe ? VideoFrameType::kVideoFrameKey
961 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100964 last_frame_types_ = frame_type;
965 }
Niels Möllerb859b322019-03-07 12:40:01 +0100966 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100967 }
968
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100969 void InjectEncodedImage(const EncodedImage& image,
970 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200971 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100972 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100973 }
974
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200975 void SetEncodedImageData(
976 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200977 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200978 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200979 }
980
Erik Språngd7329ca2019-02-21 21:19:53 +0100981 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200982 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100983 expect_null_frame_ = true;
984 }
985
Erik Språng5056af02019-09-02 15:53:11 +0200986 absl::optional<VideoEncoder::RateControlParameters>
987 GetAndResetLastRateControlSettings() {
988 auto settings = last_rate_control_settings_;
989 last_rate_control_settings_.reset();
990 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100991 }
992
Henrik Boström56db9ff2021-03-24 09:06:45 +0100993 int GetLastInputWidth() const {
994 MutexLock lock(&local_mutex_);
995 return last_input_width_;
996 }
997
998 int GetLastInputHeight() const {
999 MutexLock lock(&local_mutex_);
1000 return last_input_height_;
1001 }
1002
Evan Shrubsole895556e2020-10-05 09:15:13 +02001003 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1004 MutexLock lock(&local_mutex_);
1005 return last_input_pixel_format_;
1006 }
1007
Sergey Silkin5ee69672019-07-02 14:18:34 +02001008 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +02001009 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001010 return num_encoder_initializations_;
1011 }
1012
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001013 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001014 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001015 return num_set_rates_;
1016 }
1017
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001018 VideoCodec video_codec() const {
1019 MutexLock lock(&local_mutex_);
1020 return video_codec_;
1021 }
1022
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001023 void SetPreferredPixelFormats(
1024 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1025 pixel_formats) {
1026 MutexLock lock(&local_mutex_);
1027 preferred_pixel_formats_ = std::move(pixel_formats);
1028 }
1029
perkjfa10b552016-10-02 23:45:26 -07001030 private:
perkj26091b12016-09-01 01:17:40 -07001031 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001032 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001033 bool block_encode;
1034 {
Markus Handella3765182020-07-08 13:13:32 +02001035 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001036 if (expect_null_frame_) {
1037 EXPECT_EQ(input_image.timestamp(), 0u);
1038 EXPECT_EQ(input_image.width(), 1);
1039 last_frame_types_ = *frame_types;
1040 expect_null_frame_ = false;
1041 } else {
1042 EXPECT_GT(input_image.timestamp(), timestamp_);
1043 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1044 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1045 }
perkj26091b12016-09-01 01:17:40 -07001046
1047 timestamp_ = input_image.timestamp();
1048 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001049 last_input_width_ = input_image.width();
1050 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001051 block_encode = block_next_encode_;
1052 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001053 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001054 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001055 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001056 }
Niels Möllerb859b322019-03-07 12:40:01 +01001057 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001058 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001059 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001060
perkj26091b12016-09-01 01:17:40 -07001061 return result;
1062 }
1063
Niels Möller08ae7ce2020-09-23 15:58:12 +02001064 CodecSpecificInfo EncodeHook(
1065 EncodedImage& encoded_image,
1066 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001067 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001068 {
1069 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001070 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001071 }
1072 MutexLock lock(&local_mutex_);
1073 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001074 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001075 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001076 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001077 }
1078
sprangfe627f32017-03-29 08:24:59 -07001079 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001080 const Settings& settings) override {
1081 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001082
Markus Handella3765182020-07-08 13:13:32 +02001083 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001084 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001085
1086 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001087 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001088
Erik Språng82fad3d2018-03-21 09:57:23 +01001089 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001090 // Simulate setting up temporal layers, in order to validate the life
1091 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001092 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001093 frame_buffer_controller_ =
1094 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001095 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001096 if (force_init_encode_failed_) {
1097 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001098 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001099 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001100
Erik Språngb7cb7b52019-02-26 15:52:33 +01001101 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001102 return res;
1103 }
1104
Erik Språngb7cb7b52019-02-26 15:52:33 +01001105 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001106 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001107 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1108 initialized_ = EncoderState::kUninitialized;
1109 return FakeEncoder::Release();
1110 }
1111
Erik Språng16cb8f52019-04-12 13:59:09 +02001112 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001114 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001115 VideoBitrateAllocation adjusted_rate_allocation;
1116 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1117 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001118 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001119 adjusted_rate_allocation.SetBitrate(
1120 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001121 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001122 rate_factor_));
1123 }
1124 }
1125 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001126 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001127 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001128 RateControlParameters adjusted_paramters = parameters;
1129 adjusted_paramters.bitrate = adjusted_rate_allocation;
1130 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001131 }
1132
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001133 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001134 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001135 enum class EncoderState {
1136 kUninitialized,
1137 kInitializationFailed,
1138 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001139 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1140 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001141 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001142 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1143 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1144 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1145 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1146 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1147 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001148 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1149 false;
Markus Handella3765182020-07-08 13:13:32 +02001150 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001151 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1152 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001153 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001154 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001155 absl::optional<bool>
1156 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001157 local_mutex_);
1158 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1159 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1160 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001161 absl::optional<VideoEncoder::RateControlParameters>
1162 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001163 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1164 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001165 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001166 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001167 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1168 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001169 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001170 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001171 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001172 RTC_GUARDED_BY(local_mutex_);
1173 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001174 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001175 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1176 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001177 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1178 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001179 };
1180
mflodmancc3d4422017-08-03 08:27:51 -07001181 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001182 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001183 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1184 : time_controller_(time_controller), test_encoder_(test_encoder) {
1185 RTC_DCHECK(time_controller_);
1186 }
perkj26091b12016-09-01 01:17:40 -07001187
perkj26091b12016-09-01 01:17:40 -07001188 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001189 EXPECT_TRUE(
1190 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1191 }
1192
1193 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1194 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001195 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001196 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001197 return false;
perkj26091b12016-09-01 01:17:40 -07001198 {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001200 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001201 }
1202 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001203 return true;
perkj26091b12016-09-01 01:17:40 -07001204 }
1205
sprangb1ca0732017-02-01 08:38:12 -08001206 void WaitForEncodedFrame(uint32_t expected_width,
1207 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001208 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001209 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001210 }
1211
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001212 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001213 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001214 uint32_t width = 0;
1215 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001216 {
Markus Handella3765182020-07-08 13:13:32 +02001217 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001218 width = last_width_;
1219 height = last_height_;
1220 }
1221 EXPECT_EQ(expected_height, height);
1222 EXPECT_EQ(expected_width, width);
1223 }
1224
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001225 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1226 VideoRotation rotation;
1227 {
Markus Handella3765182020-07-08 13:13:32 +02001228 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001229 rotation = last_rotation_;
1230 }
1231 EXPECT_EQ(expected_rotation, rotation);
1232 }
1233
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001234 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001235
sprangc5d62e22017-04-02 23:53:04 -07001236 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001237 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1238 bool ret = encoded_frame_event_.Wait(timeout_ms);
1239 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1240 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001241 }
1242
perkj26091b12016-09-01 01:17:40 -07001243 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001244 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001245 expect_frames_ = false;
1246 }
1247
asaperssonfab67072017-04-04 05:51:49 -07001248 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001249 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001250 return number_of_reconfigurations_;
1251 }
1252
asaperssonfab67072017-04-04 05:51:49 -07001253 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001254 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001255 return min_transmit_bitrate_bps_;
1256 }
1257
Erik Språngd7329ca2019-02-21 21:19:53 +01001258 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001259 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001260 num_expected_layers_ = num_layers;
1261 }
1262
Erik Språngb7cb7b52019-02-26 15:52:33 +01001263 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001264 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001265 return last_capture_time_ms_;
1266 }
1267
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001268 const EncodedImage& GetLastEncodedImage() {
1269 MutexLock lock(&mutex_);
1270 return last_encoded_image_;
1271 }
1272
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001273 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001274 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001275 return std::move(last_encoded_image_data_);
1276 }
1277
Per Kjellanderdcef6412020-10-07 15:09:05 +02001278 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1279 MutexLock lock(&mutex_);
1280 return last_bitrate_allocation_;
1281 }
1282
1283 int number_of_bitrate_allocations() const {
1284 MutexLock lock(&mutex_);
1285 return number_of_bitrate_allocations_;
1286 }
1287
Per Kjellandera9434842020-10-15 17:53:22 +02001288 VideoLayersAllocation GetLastVideoLayersAllocation() {
1289 MutexLock lock(&mutex_);
1290 return last_layers_allocation_;
1291 }
1292
1293 int number_of_layers_allocations() const {
1294 MutexLock lock(&mutex_);
1295 return number_of_layers_allocations_;
1296 }
1297
perkj26091b12016-09-01 01:17:40 -07001298 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001299 Result OnEncodedImage(
1300 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001301 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001302 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001303 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001304 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001305 last_encoded_image_data_ = std::vector<uint8_t>(
1306 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001307 uint32_t timestamp = encoded_image.Timestamp();
1308 if (last_timestamp_ != timestamp) {
1309 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001310 last_width_ = encoded_image._encodedWidth;
1311 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001312 } else {
1313 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001314 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1315 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001316 }
1317 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001318 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001319 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001320 if (num_received_layers_ == num_expected_layers_) {
1321 encoded_frame_event_.Set();
1322 }
sprangb1ca0732017-02-01 08:38:12 -08001323 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001324 }
1325
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001326 void OnEncoderConfigurationChanged(
1327 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001328 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001329 VideoEncoderConfig::ContentType content_type,
1330 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001331 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001332 ++number_of_reconfigurations_;
1333 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1334 }
1335
Per Kjellanderdcef6412020-10-07 15:09:05 +02001336 void OnBitrateAllocationUpdated(
1337 const VideoBitrateAllocation& allocation) override {
1338 MutexLock lock(&mutex_);
1339 ++number_of_bitrate_allocations_;
1340 last_bitrate_allocation_ = allocation;
1341 }
1342
Per Kjellandera9434842020-10-15 17:53:22 +02001343 void OnVideoLayersAllocationUpdated(
1344 VideoLayersAllocation allocation) override {
1345 MutexLock lock(&mutex_);
1346 ++number_of_layers_allocations_;
1347 last_layers_allocation_ = allocation;
1348 rtc::StringBuilder log;
1349 for (const auto& layer : allocation.active_spatial_layers) {
1350 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1351 << "[";
1352 for (const auto target_bitrate :
1353 layer.target_bitrate_per_temporal_layer) {
1354 log << target_bitrate.kbps() << ",";
1355 }
1356 log << "]";
1357 }
1358 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1359 }
1360
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001361 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001362 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001363 TestEncoder* test_encoder_;
1364 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001365 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001366 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001367 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001368 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001369 uint32_t last_height_ = 0;
1370 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001371 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001372 size_t num_expected_layers_ = 1;
1373 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001374 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001375 int number_of_reconfigurations_ = 0;
1376 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001377 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1378 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001379 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1380 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001381 };
1382
Sergey Silkin5ee69672019-07-02 14:18:34 +02001383 class VideoBitrateAllocatorProxyFactory
1384 : public VideoBitrateAllocatorFactory {
1385 public:
1386 VideoBitrateAllocatorProxyFactory()
1387 : bitrate_allocator_factory_(
1388 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1389
1390 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1391 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001392 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001393 codec_config_ = codec;
1394 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1395 }
1396
1397 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001398 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001399 return codec_config_;
1400 }
1401
1402 private:
1403 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1404
Markus Handella3765182020-07-08 13:13:32 +02001405 mutable Mutex mutex_;
1406 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001407 };
1408
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001409 Clock* clock() { return time_controller_.GetClock(); }
1410 void AdvanceTime(TimeDelta duration) {
1411 time_controller_.AdvanceTime(duration);
1412 }
1413
1414 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1415
1416 protected:
1417 virtual TaskQueueFactory* GetTaskQueueFactory() {
1418 return time_controller_.GetTaskQueueFactory();
1419 }
1420
1421 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001422 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001423 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001424 int codec_width_;
1425 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001426 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001427 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001428 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001429 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001430 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001431 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001432 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001433 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001434};
1435
mflodmancc3d4422017-08-03 08:27:51 -07001436TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001438 DataRate::BitsPerSec(kTargetBitrateBps),
1439 DataRate::BitsPerSec(kTargetBitrateBps),
1440 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001441 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001442 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001443 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001444 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001445 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001446}
1447
mflodmancc3d4422017-08-03 08:27:51 -07001448TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001449 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001450 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001451 // The encoder will cache up to one frame for a short duration. Adding two
1452 // frames means that the first frame will be dropped and the second frame will
1453 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001454 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001455 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001456 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001457 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001458
Henrik Boström381d1092020-05-12 18:49:07 +02001459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001460 DataRate::BitsPerSec(kTargetBitrateBps),
1461 DataRate::BitsPerSec(kTargetBitrateBps),
1462 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001463
Sebastian Janssona3177052018-04-10 13:05:49 +02001464 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001465 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001466 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1467
1468 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001469 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001470}
1471
mflodmancc3d4422017-08-03 08:27:51 -07001472TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001474 DataRate::BitsPerSec(kTargetBitrateBps),
1475 DataRate::BitsPerSec(kTargetBitrateBps),
1476 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001477 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001478 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001479
Henrik Boström381d1092020-05-12 18:49:07 +02001480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1481 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1482 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001483 // The encoder will cache up to one frame for a short duration. Adding two
1484 // frames means that the first frame will be dropped and the second frame will
1485 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001486 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001487 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001488
Henrik Boström381d1092020-05-12 18:49:07 +02001489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001490 DataRate::BitsPerSec(kTargetBitrateBps),
1491 DataRate::BitsPerSec(kTargetBitrateBps),
1492 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001493 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001494 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1495 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001496 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001497}
1498
mflodmancc3d4422017-08-03 08:27:51 -07001499TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001500 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001501 DataRate::BitsPerSec(kTargetBitrateBps),
1502 DataRate::BitsPerSec(kTargetBitrateBps),
1503 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001504 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001505 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001506
1507 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001508 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001509
perkja49cbd32016-09-16 07:53:41 -07001510 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001511 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001512 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001513}
1514
mflodmancc3d4422017-08-03 08:27:51 -07001515TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001517 DataRate::BitsPerSec(kTargetBitrateBps),
1518 DataRate::BitsPerSec(kTargetBitrateBps),
1519 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001520
perkja49cbd32016-09-16 07:53:41 -07001521 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001522 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001523
mflodmancc3d4422017-08-03 08:27:51 -07001524 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001525 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001526 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001527 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1528 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001529}
1530
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001531class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1532 public:
1533 VideoStreamEncoderBlockedTest() {}
1534
1535 TaskQueueFactory* GetTaskQueueFactory() override {
1536 return task_queue_factory_.get();
1537 }
1538
1539 private:
1540 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1541 CreateDefaultTaskQueueFactory();
1542};
1543
1544TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001546 DataRate::BitsPerSec(kTargetBitrateBps),
1547 DataRate::BitsPerSec(kTargetBitrateBps),
1548 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001549
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001550 int dropped_count = 0;
1551 stats_proxy_->SetDroppedFrameCallback(
1552 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1553 ++dropped_count;
1554 });
1555
perkj26091b12016-09-01 01:17:40 -07001556 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001557 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001558 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001559 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1560 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001561 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1562 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001563 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001564 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001565
mflodmancc3d4422017-08-03 08:27:51 -07001566 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001567
1568 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001569}
1570
Henrik Boström56db9ff2021-03-24 09:06:45 +01001571TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001573 DataRate::BitsPerSec(kTargetBitrateBps),
1574 DataRate::BitsPerSec(kTargetBitrateBps),
1575 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001576
1577 rtc::Event frame_destroyed_event;
1578 video_source_.IncomingCapturedFrame(
1579 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001580 WaitForEncodedFrame(1);
1581 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1582 fake_encoder_.GetLastInputPixelFormat());
1583 EXPECT_EQ(fake_encoder_.codec_config().width,
1584 fake_encoder_.GetLastInputWidth());
1585 EXPECT_EQ(fake_encoder_.codec_config().height,
1586 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001587 video_stream_encoder_->Stop();
1588}
1589
Henrik Boström56db9ff2021-03-24 09:06:45 +01001590TEST_F(VideoStreamEncoderTest,
1591 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001592 // Use the cropping factory.
1593 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001594 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001595 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1596 kMaxPayloadLength);
1597 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1598
1599 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001600 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001601 DataRate::BitsPerSec(kTargetBitrateBps),
1602 DataRate::BitsPerSec(kTargetBitrateBps),
1603 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001604 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1605 WaitForEncodedFrame(1);
1606 // The encoder will have been configured once.
1607 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1608 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1609 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1610
1611 // Now send in a fake frame that needs to be cropped as the width/height
1612 // aren't divisible by 4 (see CreateEncoderStreams above).
1613 rtc::Event frame_destroyed_event;
1614 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1615 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001616 WaitForEncodedFrame(2);
1617 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1618 fake_encoder_.GetLastInputPixelFormat());
1619 EXPECT_EQ(fake_encoder_.codec_config().width,
1620 fake_encoder_.GetLastInputWidth());
1621 EXPECT_EQ(fake_encoder_.codec_config().height,
1622 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001623 video_stream_encoder_->Stop();
1624}
1625
Evan Shrubsole895556e2020-10-05 09:15:13 +02001626TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1628 DataRate::BitsPerSec(kTargetBitrateBps),
1629 DataRate::BitsPerSec(kTargetBitrateBps),
1630 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1631
1632 video_source_.IncomingCapturedFrame(
1633 CreateNV12Frame(1, codec_width_, codec_height_));
1634 WaitForEncodedFrame(1);
1635 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1636 fake_encoder_.GetLastInputPixelFormat());
1637 video_stream_encoder_->Stop();
1638}
1639
Henrik Boström56db9ff2021-03-24 09:06:45 +01001640TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1642 DataRate::BitsPerSec(kTargetBitrateBps),
1643 DataRate::BitsPerSec(kTargetBitrateBps),
1644 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1645
1646 fake_encoder_.SetPreferredPixelFormats({});
1647
1648 rtc::Event frame_destroyed_event;
1649 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1650 1, &frame_destroyed_event, codec_width_, codec_height_));
1651 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001652 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001653 fake_encoder_.GetLastInputPixelFormat());
1654 video_stream_encoder_->Stop();
1655}
1656
Henrik Boström56db9ff2021-03-24 09:06:45 +01001657TEST_F(VideoStreamEncoderTest,
1658 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1660 DataRate::BitsPerSec(kTargetBitrateBps),
1661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1663
1664 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1665
1666 rtc::Event frame_destroyed_event;
1667 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1668 1, &frame_destroyed_event, codec_width_, codec_height_));
1669 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001670 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001671 fake_encoder_.GetLastInputPixelFormat());
1672 video_stream_encoder_->Stop();
1673}
1674
Henrik Boström56db9ff2021-03-24 09:06:45 +01001675TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1677 DataRate::BitsPerSec(kTargetBitrateBps),
1678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1680
1681 // Fake NV12 native frame does not allow mapping to I444.
1682 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1683
1684 rtc::Event frame_destroyed_event;
1685 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1686 1, &frame_destroyed_event, codec_width_, codec_height_));
1687 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001688 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001689 fake_encoder_.GetLastInputPixelFormat());
1690 video_stream_encoder_->Stop();
1691}
1692
Henrik Boström56db9ff2021-03-24 09:06:45 +01001693TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1695 DataRate::BitsPerSec(kTargetBitrateBps),
1696 DataRate::BitsPerSec(kTargetBitrateBps),
1697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1698
1699 rtc::Event frame_destroyed_event;
1700 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1701 1, &frame_destroyed_event, codec_width_, codec_height_));
1702 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001703 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001704 fake_encoder_.GetLastInputPixelFormat());
1705 video_stream_encoder_->Stop();
1706}
1707
Ying Wang9b881ab2020-02-07 14:29:32 +01001708TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001710 DataRate::BitsPerSec(kTargetBitrateBps),
1711 DataRate::BitsPerSec(kTargetBitrateBps),
1712 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001713 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1714 WaitForEncodedFrame(1);
1715
Henrik Boström381d1092020-05-12 18:49:07 +02001716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001717 DataRate::BitsPerSec(kTargetBitrateBps),
1718 DataRate::BitsPerSec(kTargetBitrateBps),
1719 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001720 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1721 // frames. Adding two frames means that the first frame will be dropped and
1722 // the second frame will be sent to the encoder.
1723 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1724 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1725 WaitForEncodedFrame(3);
1726 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1727 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1728 WaitForEncodedFrame(5);
1729 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1730 video_stream_encoder_->Stop();
1731}
1732
mflodmancc3d4422017-08-03 08:27:51 -07001733TEST_F(VideoStreamEncoderTest,
1734 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps),
1738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001739 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001740
1741 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001742 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001743 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001744 // The encoder will have been configured once when the first frame is
1745 // received.
1746 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001747
1748 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001749 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001750 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001751 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001752 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001753
1754 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001755 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001756 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001757 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001758 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001759
mflodmancc3d4422017-08-03 08:27:51 -07001760 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001761}
1762
mflodmancc3d4422017-08-03 08:27:51 -07001763TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001764 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001765 DataRate::BitsPerSec(kTargetBitrateBps),
1766 DataRate::BitsPerSec(kTargetBitrateBps),
1767 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001768
1769 // Capture a frame and wait for it to synchronize with the encoder thread.
1770 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001771 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001772 // The encoder will have been configured once.
1773 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001774 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1775 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1776
1777 codec_width_ *= 2;
1778 codec_height_ *= 2;
1779 // Capture a frame with a higher resolution and wait for it to synchronize
1780 // with the encoder thread.
1781 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001782 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001783 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1784 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001785 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001786
mflodmancc3d4422017-08-03 08:27:51 -07001787 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001788}
1789
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001790TEST_F(VideoStreamEncoderTest,
1791 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001793 DataRate::BitsPerSec(kTargetBitrateBps),
1794 DataRate::BitsPerSec(kTargetBitrateBps),
1795 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001796
1797 // Capture a frame and wait for it to synchronize with the encoder thread.
1798 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1799 WaitForEncodedFrame(1);
1800
1801 VideoEncoderConfig video_encoder_config;
1802 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1803 // Changing the max payload data length recreates encoder.
1804 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1805 kMaxPayloadLength / 2);
1806
1807 // Capture a frame and wait for it to synchronize with the encoder thread.
1808 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1809 WaitForEncodedFrame(2);
1810 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1811
1812 video_stream_encoder_->Stop();
1813}
1814
Sergey Silkin5ee69672019-07-02 14:18:34 +02001815TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001817 DataRate::BitsPerSec(kTargetBitrateBps),
1818 DataRate::BitsPerSec(kTargetBitrateBps),
1819 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001820
1821 VideoEncoderConfig video_encoder_config;
1822 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1823 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1824 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1825 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1826 kMaxPayloadLength);
1827
1828 // Capture a frame and wait for it to synchronize with the encoder thread.
1829 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1830 WaitForEncodedFrame(1);
1831 // The encoder will have been configured once when the first frame is
1832 // received.
1833 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1834 EXPECT_EQ(kTargetBitrateBps,
1835 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1836 EXPECT_EQ(kStartBitrateBps,
1837 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1838
Sergey Silkin6456e352019-07-08 17:56:40 +02001839 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1840 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001841 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1842 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1843 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1844 kMaxPayloadLength);
1845
1846 // Capture a frame and wait for it to synchronize with the encoder thread.
1847 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1848 WaitForEncodedFrame(2);
1849 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1850 // Bitrate limits have changed - rate allocator should be reconfigured,
1851 // encoder should not be reconfigured.
1852 EXPECT_EQ(kTargetBitrateBps * 2,
1853 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1854 EXPECT_EQ(kStartBitrateBps * 2,
1855 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1856 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1857
1858 video_stream_encoder_->Stop();
1859}
1860
Sergey Silkin6456e352019-07-08 17:56:40 +02001861TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001862 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001864 DataRate::BitsPerSec(kTargetBitrateBps),
1865 DataRate::BitsPerSec(kTargetBitrateBps),
1866 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001867
Sergey Silkincd02eba2020-01-20 14:48:40 +01001868 const uint32_t kMinEncBitrateKbps = 100;
1869 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001870 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001871 /*frame_size_pixels=*/codec_width_ * codec_height_,
1872 /*min_start_bitrate_bps=*/0,
1873 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1874 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001875 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1876
Sergey Silkincd02eba2020-01-20 14:48:40 +01001877 VideoEncoderConfig video_encoder_config;
1878 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1879 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1880 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1881 (kMinEncBitrateKbps + 1) * 1000;
1882 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1883 kMaxPayloadLength);
1884
1885 // When both encoder and app provide bitrate limits, the intersection of
1886 // provided sets should be used.
1887 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1888 WaitForEncodedFrame(1);
1889 EXPECT_EQ(kMaxEncBitrateKbps,
1890 bitrate_allocator_factory_.codec_config().maxBitrate);
1891 EXPECT_EQ(kMinEncBitrateKbps + 1,
1892 bitrate_allocator_factory_.codec_config().minBitrate);
1893
1894 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1895 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1896 (kMinEncBitrateKbps - 1) * 1000;
1897 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1898 kMaxPayloadLength);
1899 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001901 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001902 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001903 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001904 bitrate_allocator_factory_.codec_config().minBitrate);
1905
Sergey Silkincd02eba2020-01-20 14:48:40 +01001906 video_stream_encoder_->Stop();
1907}
1908
1909TEST_F(VideoStreamEncoderTest,
1910 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001911 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001912 DataRate::BitsPerSec(kTargetBitrateBps),
1913 DataRate::BitsPerSec(kTargetBitrateBps),
1914 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001915
1916 const uint32_t kMinAppBitrateKbps = 100;
1917 const uint32_t kMaxAppBitrateKbps = 200;
1918 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1919 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1920 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1921 /*frame_size_pixels=*/codec_width_ * codec_height_,
1922 /*min_start_bitrate_bps=*/0,
1923 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1924 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1925 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1926
1927 VideoEncoderConfig video_encoder_config;
1928 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1929 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1930 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1931 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1933 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001934
Sergey Silkincd02eba2020-01-20 14:48:40 +01001935 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1936 WaitForEncodedFrame(1);
1937 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001938 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001939 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001940 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001941
1942 video_stream_encoder_->Stop();
1943}
1944
1945TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001946 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001948 DataRate::BitsPerSec(kTargetBitrateBps),
1949 DataRate::BitsPerSec(kTargetBitrateBps),
1950 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001951
1952 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001953 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001954 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001955 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001956 fake_encoder_.SetResolutionBitrateLimits(
1957 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1958
1959 VideoEncoderConfig video_encoder_config;
1960 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1961 video_encoder_config.max_bitrate_bps = 0;
1962 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1963 kMaxPayloadLength);
1964
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001965 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001966 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1967 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001968 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1969 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001970 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1971 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1972
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001973 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001974 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1975 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001976 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1977 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001978 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1979 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1980
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001981 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001982 // encoder for 360p should be used.
1983 video_source_.IncomingCapturedFrame(
1984 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1985 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001986 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1987 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001988 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1989 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1990
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001992 // ignored.
1993 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1994 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001997 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1998 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001999 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2000 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002001 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2002 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2003
2004 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2005 // for 270p should be used.
2006 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2007 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2009 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002010 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2011 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2012
2013 video_stream_encoder_->Stop();
2014}
2015
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002016TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002018 DataRate::BitsPerSec(kTargetBitrateBps),
2019 DataRate::BitsPerSec(kTargetBitrateBps),
2020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002021
2022 VideoEncoderConfig video_encoder_config;
2023 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2024 video_encoder_config.max_bitrate_bps = 0;
2025 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2026 kMaxPayloadLength);
2027
2028 // Encode 720p frame to get the default encoder target bitrate.
2029 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2030 WaitForEncodedFrame(1);
2031 const uint32_t kDefaultTargetBitrateFor720pKbps =
2032 bitrate_allocator_factory_.codec_config()
2033 .simulcastStream[0]
2034 .targetBitrate;
2035
2036 // Set the max recommended encoder bitrate to something lower than the default
2037 // target bitrate.
2038 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2039 1280 * 720, 10 * 1000, 10 * 1000,
2040 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2041 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2042
2043 // Change resolution to trigger encoder reinitialization.
2044 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2045 WaitForEncodedFrame(2);
2046 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2047 WaitForEncodedFrame(3);
2048
2049 // Ensure the target bitrate is capped by the max bitrate.
2050 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2051 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2052 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2053 .simulcastStream[0]
2054 .targetBitrate *
2055 1000,
2056 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2057
2058 video_stream_encoder_->Stop();
2059}
2060
Åsa Perssona7e34d32021-01-20 15:36:13 +01002061TEST_F(VideoStreamEncoderTest,
2062 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2063 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2064 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2065 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2066 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2067 fake_encoder_.SetResolutionBitrateLimits(
2068 {kEncoderLimits270p, kEncoderLimits360p});
2069
2070 // Two streams, highest stream active.
2071 VideoEncoderConfig config;
2072 const int kNumStreams = 2;
2073 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2074 config.max_bitrate_bps = 0;
2075 config.simulcast_layers[0].active = false;
2076 config.simulcast_layers[1].active = true;
2077 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002078 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002079 "VP8", /*max qp*/ 56, /*screencast*/ false,
2080 /*screenshare enabled*/ false);
2081 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2082
2083 // The encoder bitrate limits for 270p should be used.
2084 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2085 EXPECT_FALSE(WaitForFrame(1000));
2086 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2087 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2088 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2089 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2090 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2091
2092 // The encoder bitrate limits for 360p should be used.
2093 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2094 EXPECT_FALSE(WaitForFrame(1000));
2095 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2096 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2097 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2098 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2099
2100 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2101 video_source_.IncomingCapturedFrame(
2102 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2103 EXPECT_FALSE(WaitForFrame(1000));
2104 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2105 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2106 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2107 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2108
2109 // Resolution higher than 360p. Encoder limits should be ignored.
2110 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2111 EXPECT_FALSE(WaitForFrame(1000));
2112 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2113 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2114 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2115 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2116 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2117 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2118 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2119 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2120
2121 // Resolution lower than 270p. The encoder limits for 270p should be used.
2122 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2123 EXPECT_FALSE(WaitForFrame(1000));
2124 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2125 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2126 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2127 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2128
2129 video_stream_encoder_->Stop();
2130}
2131
2132TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002133 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2134 // Two streams, highest stream active.
2135 VideoEncoderConfig config;
2136 const int kNumStreams = 2;
2137 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2138 config.max_bitrate_bps = 0;
2139 config.simulcast_layers[0].active = false;
2140 config.simulcast_layers[1].active = true;
2141 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002142 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002143 "VP8", /*max qp*/ 56, /*screencast*/ false,
2144 /*screenshare enabled*/ false);
2145 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2146
2147 // Default bitrate limits for 270p should be used.
2148 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2149 kDefaultLimits270p =
2150 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002151 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002152 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2153 EXPECT_FALSE(WaitForFrame(1000));
2154 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2155 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2156 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2157 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2158 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2159
2160 // Default bitrate limits for 360p should be used.
2161 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2162 kDefaultLimits360p =
2163 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002164 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002165 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2166 EXPECT_FALSE(WaitForFrame(1000));
2167 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2168 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2169 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2170 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2171
2172 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2173 video_source_.IncomingCapturedFrame(
2174 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2177 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2178 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2179 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2180
2181 // Default bitrate limits for 540p should be used.
2182 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2183 kDefaultLimits540p =
2184 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002185 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002186 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2187 EXPECT_FALSE(WaitForFrame(1000));
2188 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2189 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2190 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2191 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2192
2193 video_stream_encoder_->Stop();
2194}
2195
2196TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002197 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2198 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2199 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2200 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2201 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2202 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2203 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2204 fake_encoder_.SetResolutionBitrateLimits(
2205 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2206
2207 // Three streams, middle stream active.
2208 VideoEncoderConfig config;
2209 const int kNumStreams = 3;
2210 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2211 config.simulcast_layers[0].active = false;
2212 config.simulcast_layers[1].active = true;
2213 config.simulcast_layers[2].active = false;
2214 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002215 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216 "VP8", /*max qp*/ 56, /*screencast*/ false,
2217 /*screenshare enabled*/ false);
2218 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2219
2220 // The encoder bitrate limits for 360p should be used.
2221 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2222 EXPECT_FALSE(WaitForFrame(1000));
2223 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2224 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2225 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2226 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2227 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2228
2229 // The encoder bitrate limits for 270p should be used.
2230 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2231 EXPECT_FALSE(WaitForFrame(1000));
2232 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2233 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2234 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2235 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2236
2237 video_stream_encoder_->Stop();
2238}
2239
2240TEST_F(VideoStreamEncoderTest,
2241 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2242 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2243 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2244 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2245 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2246 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2247 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2248 fake_encoder_.SetResolutionBitrateLimits(
2249 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2250
2251 // Three streams, lowest stream active.
2252 VideoEncoderConfig config;
2253 const int kNumStreams = 3;
2254 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2255 config.simulcast_layers[0].active = true;
2256 config.simulcast_layers[1].active = false;
2257 config.simulcast_layers[2].active = false;
2258 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002259 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002260 "VP8", /*max qp*/ 56, /*screencast*/ false,
2261 /*screenshare enabled*/ false);
2262 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2263
2264 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2265 // on lowest stream, limits for 270p should not be used
2266 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2267 EXPECT_FALSE(WaitForFrame(1000));
2268 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2269 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2270 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2271 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2272 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2273
2274 video_stream_encoder_->Stop();
2275}
2276
2277TEST_F(VideoStreamEncoderTest,
2278 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2279 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2280 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2281 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2282 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2283 fake_encoder_.SetResolutionBitrateLimits(
2284 {kEncoderLimits270p, kEncoderLimits360p});
2285 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2286
2287 // Two streams, highest stream active.
2288 VideoEncoderConfig config;
2289 const int kNumStreams = 2;
2290 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2291 config.simulcast_layers[0].active = false;
2292 config.simulcast_layers[1].active = true;
2293 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2294 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002295 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002296 "VP8", /*max qp*/ 56, /*screencast*/ false,
2297 /*screenshare enabled*/ false);
2298 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2299
2300 // The encoder bitrate limits for 270p should be used.
2301 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2302 EXPECT_FALSE(WaitForFrame(1000));
2303 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2304 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2305 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2306 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2307 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2308
2309 // The max configured bitrate is less than the encoder limit for 360p.
2310 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2311 EXPECT_FALSE(WaitForFrame(1000));
2312 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2313 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2314 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2315 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2316
2317 video_stream_encoder_->Stop();
2318}
2319
mflodmancc3d4422017-08-03 08:27:51 -07002320TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002321 EXPECT_TRUE(video_source_.has_sinks());
2322 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002323 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002324 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002325 EXPECT_FALSE(video_source_.has_sinks());
2326 EXPECT_TRUE(new_video_source.has_sinks());
2327
mflodmancc3d4422017-08-03 08:27:51 -07002328 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002329}
2330
mflodmancc3d4422017-08-03 08:27:51 -07002331TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002332 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002333 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002334 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002335 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002336}
2337
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002338class ResolutionAlignmentTest
2339 : public VideoStreamEncoderTest,
2340 public ::testing::WithParamInterface<
2341 ::testing::tuple<int, std::vector<double>>> {
2342 public:
2343 ResolutionAlignmentTest()
2344 : requested_alignment_(::testing::get<0>(GetParam())),
2345 scale_factors_(::testing::get<1>(GetParam())) {}
2346
2347 protected:
2348 const int requested_alignment_;
2349 const std::vector<double> scale_factors_;
2350};
2351
2352INSTANTIATE_TEST_SUITE_P(
2353 AlignmentAndScaleFactors,
2354 ResolutionAlignmentTest,
2355 ::testing::Combine(
2356 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2357 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2358 std::vector<double>{-1.0, -1.0},
2359 std::vector<double>{-1.0, -1.0, -1.0},
2360 std::vector<double>{4.0, 2.0, 1.0},
2361 std::vector<double>{9999.0, -1.0, 1.0},
2362 std::vector<double>{3.99, 2.01, 1.0},
2363 std::vector<double>{4.9, 1.7, 1.25},
2364 std::vector<double>{10.0, 4.0, 3.0},
2365 std::vector<double>{1.75, 3.5},
2366 std::vector<double>{1.5, 2.5},
2367 std::vector<double>{1.3, 1.0})));
2368
2369TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2370 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002371 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002372 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2373 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2374
2375 // Fill config with the scaling factor by which to reduce encoding size.
2376 const int num_streams = scale_factors_.size();
2377 VideoEncoderConfig config;
2378 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2379 for (int i = 0; i < num_streams; ++i) {
2380 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2381 }
2382 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002383 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002384 "VP8", /*max qp*/ 56, /*screencast*/ false,
2385 /*screenshare enabled*/ false);
2386 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2387
Henrik Boström381d1092020-05-12 18:49:07 +02002388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002389 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2390 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2391 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2392 // Wait for all layers before triggering event.
2393 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002394
2395 // On the 1st frame, we should have initialized the encoder and
2396 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002397 int64_t timestamp_ms = kFrameIntervalMs;
2398 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2399 WaitForEncodedFrame(timestamp_ms);
2400 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002401
2402 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2403 // (It's up the to the encoder to potentially drop the previous frame,
2404 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002405 timestamp_ms += kFrameIntervalMs;
2406 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2407 WaitForEncodedFrame(timestamp_ms);
2408 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2409
2410 VideoCodec codec = fake_encoder_.video_codec();
2411 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2412 // Frame size should be a multiple of the requested alignment.
2413 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2414 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2415 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2416 // Aspect ratio should match.
2417 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2418 codec.height * codec.simulcastStream[i].width);
2419 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002420
2421 video_stream_encoder_->Stop();
2422}
2423
Jonathan Yubc771b72017-12-08 17:04:29 -08002424TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2425 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002426 const int kWidth = 1280;
2427 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002428
2429 // We rely on the automatic resolution adaptation, but we handle framerate
2430 // adaptation manually by mocking the stats proxy.
2431 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002432
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002433 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002435 DataRate::BitsPerSec(kTargetBitrateBps),
2436 DataRate::BitsPerSec(kTargetBitrateBps),
2437 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002438 video_stream_encoder_->SetSource(&video_source_,
2439 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002440 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002441 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002443 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2444
Jonathan Yubc771b72017-12-08 17:04:29 -08002445 // Adapt down as far as possible.
2446 rtc::VideoSinkWants last_wants;
2447 int64_t t = 1;
2448 int loop_count = 0;
2449 do {
2450 ++loop_count;
2451 last_wants = video_source_.sink_wants();
2452
2453 // Simulate the framerate we've been asked to adapt to.
2454 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2455 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2456 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2457 mock_stats.input_frame_rate = fps;
2458 stats_proxy_->SetMockStats(mock_stats);
2459
2460 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2461 sink_.WaitForEncodedFrame(t);
2462 t += frame_interval_ms;
2463
mflodmancc3d4422017-08-03 08:27:51 -07002464 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002465 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002466 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002467 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2468 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002469 } while (video_source_.sink_wants().max_pixel_count <
2470 last_wants.max_pixel_count ||
2471 video_source_.sink_wants().max_framerate_fps <
2472 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002473
Jonathan Yubc771b72017-12-08 17:04:29 -08002474 // Verify that we've adapted all the way down.
2475 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002476 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002477 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2478 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002479 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002480 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2481 *video_source_.last_sent_height());
2482 EXPECT_EQ(kMinBalancedFramerateFps,
2483 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002484
Jonathan Yubc771b72017-12-08 17:04:29 -08002485 // Adapt back up the same number of times we adapted down.
2486 for (int i = 0; i < loop_count - 1; ++i) {
2487 last_wants = video_source_.sink_wants();
2488
2489 // Simulate the framerate we've been asked to adapt to.
2490 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2491 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2492 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2493 mock_stats.input_frame_rate = fps;
2494 stats_proxy_->SetMockStats(mock_stats);
2495
2496 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2497 sink_.WaitForEncodedFrame(t);
2498 t += frame_interval_ms;
2499
Henrik Boström91aa7322020-04-28 12:24:33 +02002500 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002501 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002502 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002503 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2504 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002505 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2506 last_wants.max_pixel_count ||
2507 video_source_.sink_wants().max_framerate_fps >
2508 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002509 }
2510
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002511 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002512 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002513 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002514 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2515 EXPECT_EQ((loop_count - 1) * 2,
2516 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002517
mflodmancc3d4422017-08-03 08:27:51 -07002518 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002519}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002520
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002521TEST_F(VideoStreamEncoderTest,
2522 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2523 video_stream_encoder_->OnBitrateUpdated(
2524 DataRate::BitsPerSec(kTargetBitrateBps),
2525 DataRate::BitsPerSec(kTargetBitrateBps),
2526 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002527 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002528
2529 const int kFrameWidth = 1280;
2530 const int kFrameHeight = 720;
2531
2532 int64_t ntp_time = kFrameIntervalMs;
2533
2534 // Force an input frame rate to be available, or the adaptation call won't
2535 // know what framerate to adapt form.
2536 const int kInputFps = 30;
2537 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2538 stats.input_frame_rate = kInputFps;
2539 stats_proxy_->SetMockStats(stats);
2540
2541 video_source_.set_adaptation_enabled(true);
2542 video_stream_encoder_->SetSource(
2543 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002544 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002545 video_source_.IncomingCapturedFrame(
2546 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2547 sink_.WaitForEncodedFrame(ntp_time);
2548 ntp_time += kFrameIntervalMs;
2549
2550 // Trigger CPU overuse.
2551 video_stream_encoder_->TriggerCpuOveruse();
2552 video_source_.IncomingCapturedFrame(
2553 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2554 sink_.WaitForEncodedFrame(ntp_time);
2555 ntp_time += kFrameIntervalMs;
2556
2557 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2558 EXPECT_EQ(std::numeric_limits<int>::max(),
2559 video_source_.sink_wants().max_pixel_count);
2560 // Some framerate constraint should be set.
2561 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2562 EXPECT_LT(restricted_fps, kInputFps);
2563 video_source_.IncomingCapturedFrame(
2564 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2565 sink_.WaitForEncodedFrame(ntp_time);
2566 ntp_time += 100;
2567
Henrik Boström2671dac2020-05-19 16:29:09 +02002568 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002569 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2570 // Give the encoder queue time to process the change in degradation preference
2571 // by waiting for an encoded frame.
2572 video_source_.IncomingCapturedFrame(
2573 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2574 sink_.WaitForEncodedFrame(ntp_time);
2575 ntp_time += kFrameIntervalMs;
2576
2577 video_stream_encoder_->TriggerQualityLow();
2578 video_source_.IncomingCapturedFrame(
2579 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2580 sink_.WaitForEncodedFrame(ntp_time);
2581 ntp_time += kFrameIntervalMs;
2582
2583 // Some resolution constraint should be set.
2584 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2585 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2586 kFrameWidth * kFrameHeight);
2587 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2588
2589 int pixel_count = video_source_.sink_wants().max_pixel_count;
2590 // Triggering a CPU underuse should not change the sink wants since it has
2591 // not been overused for resolution since we changed degradation preference.
2592 video_stream_encoder_->TriggerCpuUnderuse();
2593 video_source_.IncomingCapturedFrame(
2594 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2595 sink_.WaitForEncodedFrame(ntp_time);
2596 ntp_time += kFrameIntervalMs;
2597 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2598 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2599
Evan Shrubsole64469032020-06-11 10:45:29 +02002600 // Change the degradation preference back. CPU underuse should not adapt since
2601 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002602 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002603 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2604 video_source_.IncomingCapturedFrame(
2605 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2606 sink_.WaitForEncodedFrame(ntp_time);
2607 ntp_time += 100;
2608 // Resolution adaptations is gone after changing degradation preference.
2609 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2610 EXPECT_EQ(std::numeric_limits<int>::max(),
2611 video_source_.sink_wants().max_pixel_count);
2612 // The fps adaptation from above is now back.
2613 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2614
2615 // Trigger CPU underuse.
2616 video_stream_encoder_->TriggerCpuUnderuse();
2617 video_source_.IncomingCapturedFrame(
2618 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2619 sink_.WaitForEncodedFrame(ntp_time);
2620 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002621 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2622
2623 // Trigger QP underuse, fps should return to normal.
2624 video_stream_encoder_->TriggerQualityHigh();
2625 video_source_.IncomingCapturedFrame(
2626 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2627 sink_.WaitForEncodedFrame(ntp_time);
2628 ntp_time += kFrameIntervalMs;
2629 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002630
2631 video_stream_encoder_->Stop();
2632}
2633
mflodmancc3d4422017-08-03 08:27:51 -07002634TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002636 DataRate::BitsPerSec(kTargetBitrateBps),
2637 DataRate::BitsPerSec(kTargetBitrateBps),
2638 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002639 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002640
sprangc5d62e22017-04-02 23:53:04 -07002641 const int kFrameWidth = 1280;
2642 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002643
Åsa Persson8c1bf952018-09-13 10:42:19 +02002644 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002645
kthelgason5e13d412016-12-01 03:59:51 -08002646 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002647 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002648 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002649 frame_timestamp += kFrameIntervalMs;
2650
perkj803d97f2016-11-01 11:45:46 -07002651 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002652 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002653 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002654 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002655 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002656 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002657
asapersson0944a802017-04-07 00:57:58 -07002658 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002659 // wanted resolution.
2660 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2661 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2662 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002663 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002664
2665 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002666 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002667 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002668 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002669 // Give the encoder queue time to process the change in degradation preference
2670 // by waiting for an encoded frame.
2671 new_video_source.IncomingCapturedFrame(
2672 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2673 sink_.WaitForEncodedFrame(frame_timestamp);
2674 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002675 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002676 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002677
sprangc5d62e22017-04-02 23:53:04 -07002678 // Force an input frame rate to be available, or the adaptation call won't
2679 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002680 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002681 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002682 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002683 stats_proxy_->SetMockStats(stats);
2684
mflodmancc3d4422017-08-03 08:27:51 -07002685 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002686 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002687 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002688 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002689 frame_timestamp += kFrameIntervalMs;
2690
2691 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002692 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002693 EXPECT_EQ(std::numeric_limits<int>::max(),
2694 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002695 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002696
asapersson02465b82017-04-10 01:12:52 -07002697 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002698 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2699 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002700 // Give the encoder queue time to process the change in degradation preference
2701 // by waiting for an encoded frame.
2702 new_video_source.IncomingCapturedFrame(
2703 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2704 sink_.WaitForEncodedFrame(frame_timestamp);
2705 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002706 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002707
mflodmancc3d4422017-08-03 08:27:51 -07002708 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002709 new_video_source.IncomingCapturedFrame(
2710 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002711 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002712 frame_timestamp += kFrameIntervalMs;
2713
2714 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002715 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002716
2717 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002718 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002719 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002720 // Give the encoder queue time to process the change in degradation preference
2721 // by waiting for an encoded frame.
2722 new_video_source.IncomingCapturedFrame(
2723 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2724 sink_.WaitForEncodedFrame(frame_timestamp);
2725 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002726 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2727 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002728 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002729 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002730
2731 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002732 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002733 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002734 // Give the encoder queue time to process the change in degradation preference
2735 // by waiting for an encoded frame.
2736 new_video_source.IncomingCapturedFrame(
2737 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2738 sink_.WaitForEncodedFrame(frame_timestamp);
2739 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002740 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2741 EXPECT_EQ(std::numeric_limits<int>::max(),
2742 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002743 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002744
mflodmancc3d4422017-08-03 08:27:51 -07002745 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002746}
2747
mflodmancc3d4422017-08-03 08:27:51 -07002748TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002750 DataRate::BitsPerSec(kTargetBitrateBps),
2751 DataRate::BitsPerSec(kTargetBitrateBps),
2752 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002753
asaperssonfab67072017-04-04 05:51:49 -07002754 const int kWidth = 1280;
2755 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002756 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002757 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002758 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2759 EXPECT_FALSE(stats.bw_limited_resolution);
2760 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2761
2762 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002763 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002764 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002765 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002766
2767 stats = stats_proxy_->GetStats();
2768 EXPECT_TRUE(stats.bw_limited_resolution);
2769 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2770
2771 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002772 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002773 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002774 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002775
2776 stats = stats_proxy_->GetStats();
2777 EXPECT_FALSE(stats.bw_limited_resolution);
2778 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2779 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2780
mflodmancc3d4422017-08-03 08:27:51 -07002781 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002782}
2783
mflodmancc3d4422017-08-03 08:27:51 -07002784TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002785 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002786 DataRate::BitsPerSec(kTargetBitrateBps),
2787 DataRate::BitsPerSec(kTargetBitrateBps),
2788 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002789
2790 const int kWidth = 1280;
2791 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002792 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002793 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002794 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2795 EXPECT_FALSE(stats.cpu_limited_resolution);
2796 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2797
2798 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002800 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002801 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002802
2803 stats = stats_proxy_->GetStats();
2804 EXPECT_TRUE(stats.cpu_limited_resolution);
2805 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2806
2807 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002808 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002809 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002810 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002811
2812 stats = stats_proxy_->GetStats();
2813 EXPECT_FALSE(stats.cpu_limited_resolution);
2814 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002815 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002816
mflodmancc3d4422017-08-03 08:27:51 -07002817 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002818}
2819
mflodmancc3d4422017-08-03 08:27:51 -07002820TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002822 DataRate::BitsPerSec(kTargetBitrateBps),
2823 DataRate::BitsPerSec(kTargetBitrateBps),
2824 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002825
asaperssonfab67072017-04-04 05:51:49 -07002826 const int kWidth = 1280;
2827 const int kHeight = 720;
2828 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002829 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002830 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002831 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002832 EXPECT_FALSE(stats.cpu_limited_resolution);
2833 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2834
asaperssonfab67072017-04-04 05:51:49 -07002835 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002836 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002837 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002838 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002839 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002840 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002841 EXPECT_TRUE(stats.cpu_limited_resolution);
2842 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2843
2844 // Set new source with adaptation still enabled.
2845 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002846 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002847 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002848
asaperssonfab67072017-04-04 05:51:49 -07002849 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002850 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002851 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002852 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002853 EXPECT_TRUE(stats.cpu_limited_resolution);
2854 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2855
2856 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002857 video_stream_encoder_->SetSource(&new_video_source,
2858 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002859
asaperssonfab67072017-04-04 05:51:49 -07002860 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002861 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002862 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002863 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002864 EXPECT_FALSE(stats.cpu_limited_resolution);
2865 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2866
2867 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002868 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002869 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002870
asaperssonfab67072017-04-04 05:51:49 -07002871 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002872 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002873 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002874 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002875 EXPECT_TRUE(stats.cpu_limited_resolution);
2876 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2877
asaperssonfab67072017-04-04 05:51:49 -07002878 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002879 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002880 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002881 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002882 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002883 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002884 EXPECT_FALSE(stats.cpu_limited_resolution);
2885 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002886 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002887
mflodmancc3d4422017-08-03 08:27:51 -07002888 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002889}
2890
mflodmancc3d4422017-08-03 08:27:51 -07002891TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002893 DataRate::BitsPerSec(kTargetBitrateBps),
2894 DataRate::BitsPerSec(kTargetBitrateBps),
2895 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002896
asaperssonfab67072017-04-04 05:51:49 -07002897 const int kWidth = 1280;
2898 const int kHeight = 720;
2899 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002900 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002901 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002902 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002903 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002904 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002905
2906 // Set new source with adaptation still enabled.
2907 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002908 video_stream_encoder_->SetSource(&new_video_source,
2909 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002910
asaperssonfab67072017-04-04 05:51:49 -07002911 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002912 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002913 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002914 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002915 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002916 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002917
asaperssonfab67072017-04-04 05:51:49 -07002918 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002919 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002920 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002922 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002923 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002924 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002925 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002926
asaperssonfab67072017-04-04 05:51:49 -07002927 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002928 video_stream_encoder_->SetSource(&new_video_source,
2929 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002930
asaperssonfab67072017-04-04 05:51:49 -07002931 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002932 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002933 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002934 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002935 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002936 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002937
asapersson02465b82017-04-10 01:12:52 -07002938 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002939 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002940 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002941
asaperssonfab67072017-04-04 05:51:49 -07002942 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002943 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002944 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002945 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002946 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002947 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2948 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002949
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002951}
2952
mflodmancc3d4422017-08-03 08:27:51 -07002953TEST_F(VideoStreamEncoderTest,
2954 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002955 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002956 DataRate::BitsPerSec(kTargetBitrateBps),
2957 DataRate::BitsPerSec(kTargetBitrateBps),
2958 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002959
2960 const int kWidth = 1280;
2961 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002962 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002963 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002964 video_source_.IncomingCapturedFrame(
2965 CreateFrame(timestamp_ms, kWidth, kHeight));
2966 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002967 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2968 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2969 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2970
2971 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002973 timestamp_ms += kFrameIntervalMs;
2974 video_source_.IncomingCapturedFrame(
2975 CreateFrame(timestamp_ms, kWidth, kHeight));
2976 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002977 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2978 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2979 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2980
2981 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002982 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002983 timestamp_ms += kFrameIntervalMs;
2984 video_source_.IncomingCapturedFrame(
2985 CreateFrame(timestamp_ms, kWidth, kHeight));
2986 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002987 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2988 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2989 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2990
Niels Möller4db138e2018-04-19 09:04:13 +02002991 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002992 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002993
2994 VideoEncoderConfig video_encoder_config;
2995 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2996 // Make format different, to force recreation of encoder.
2997 video_encoder_config.video_format.parameters["foo"] = "foo";
2998 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002999 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003000 timestamp_ms += kFrameIntervalMs;
3001 video_source_.IncomingCapturedFrame(
3002 CreateFrame(timestamp_ms, kWidth, kHeight));
3003 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003004 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3005 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3006 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3007
mflodmancc3d4422017-08-03 08:27:51 -07003008 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003009}
3010
mflodmancc3d4422017-08-03 08:27:51 -07003011TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003012 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003014 DataRate::BitsPerSec(kTargetBitrateBps),
3015 DataRate::BitsPerSec(kTargetBitrateBps),
3016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3017
3018 const int kWidth = 1280;
3019 const int kHeight = 720;
3020 int sequence = 1;
3021
3022 // Enable BALANCED preference, no initial limitation.
3023 test::FrameForwarder source;
3024 video_stream_encoder_->SetSource(&source,
3025 webrtc::DegradationPreference::BALANCED);
3026 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3027 WaitForEncodedFrame(sequence++);
3028 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3029 EXPECT_FALSE(stats.cpu_limited_resolution);
3030 EXPECT_FALSE(stats.cpu_limited_framerate);
3031 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3032
3033 // Trigger CPU overuse, should now adapt down.
3034 video_stream_encoder_->TriggerCpuOveruse();
3035 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3036 WaitForEncodedFrame(sequence++);
3037 stats = stats_proxy_->GetStats();
3038 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3039
3040 // Set new degradation preference should clear restrictions since we changed
3041 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003042 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003043 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 stats = stats_proxy_->GetStats();
3047 EXPECT_FALSE(stats.cpu_limited_resolution);
3048 EXPECT_FALSE(stats.cpu_limited_framerate);
3049 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3050
3051 // Force an input frame rate to be available, or the adaptation call won't
3052 // know what framerate to adapt from.
3053 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3054 mock_stats.input_frame_rate = 30;
3055 stats_proxy_->SetMockStats(mock_stats);
3056 video_stream_encoder_->TriggerCpuOveruse();
3057 stats_proxy_->ResetMockStats();
3058 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3059 WaitForEncodedFrame(sequence++);
3060
3061 // We have now adapted once.
3062 stats = stats_proxy_->GetStats();
3063 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3064
3065 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003066 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3067 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003068 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3069 WaitForEncodedFrame(sequence++);
3070 stats = stats_proxy_->GetStats();
3071 EXPECT_FALSE(stats.cpu_limited_resolution);
3072 EXPECT_FALSE(stats.cpu_limited_framerate);
3073 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3074
3075 video_stream_encoder_->Stop();
3076}
3077
3078TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003079 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003081 DataRate::BitsPerSec(kTargetBitrateBps),
3082 DataRate::BitsPerSec(kTargetBitrateBps),
3083 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003084
asapersson0944a802017-04-07 00:57:58 -07003085 const int kWidth = 1280;
3086 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003087 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003088
asaperssonfab67072017-04-04 05:51:49 -07003089 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003090 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003091 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003092 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003093 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003094 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3095
asapersson02465b82017-04-10 01:12:52 -07003096 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003097 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003098 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003099 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003100 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003101 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003102 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003103 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3104
3105 // Set new source with adaptation still enabled.
3106 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003107 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003108 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003109
3110 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003111 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003112 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003113 stats = stats_proxy_->GetStats();
3114 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003115 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003116 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3117
sprangc5d62e22017-04-02 23:53:04 -07003118 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003119 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003120 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003121 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003122 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003123 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003124 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003125 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003126 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003127 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003128 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3129
sprangc5d62e22017-04-02 23:53:04 -07003130 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003131 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003132 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3133 mock_stats.input_frame_rate = 30;
3134 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003135 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003136 stats_proxy_->ResetMockStats();
3137
3138 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003139 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003140 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003141
3142 // Framerate now adapted.
3143 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003144 EXPECT_FALSE(stats.cpu_limited_resolution);
3145 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003146 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3147
3148 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003149 video_stream_encoder_->SetSource(&new_video_source,
3150 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003151 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003152 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003153 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003154
3155 stats = stats_proxy_->GetStats();
3156 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003157 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003158 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3159
3160 // Try to trigger overuse. Should not succeed.
3161 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003162 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003163 stats_proxy_->ResetMockStats();
3164
3165 stats = stats_proxy_->GetStats();
3166 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003167 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003168 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3169
3170 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003171 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003172 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003173 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003174 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003175 stats = stats_proxy_->GetStats();
3176 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003177 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003178 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003179
3180 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003181 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003182 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003184 stats = stats_proxy_->GetStats();
3185 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003186 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003187 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3188
3189 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003190 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003191 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003192 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003193 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003194 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003195 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003196 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003197 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003198 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003199 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3200
3201 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003202 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003203 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003204 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003205 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003206 stats = stats_proxy_->GetStats();
3207 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003208 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003209 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003210 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003211
mflodmancc3d4422017-08-03 08:27:51 -07003212 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003213}
3214
mflodmancc3d4422017-08-03 08:27:51 -07003215TEST_F(VideoStreamEncoderTest,
3216 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003217 const int kWidth = 1280;
3218 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003220 DataRate::BitsPerSec(kTargetBitrateBps),
3221 DataRate::BitsPerSec(kTargetBitrateBps),
3222 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003223
asaperssonfab67072017-04-04 05:51:49 -07003224 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003225 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003226
asaperssonfab67072017-04-04 05:51:49 -07003227 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003228 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003229
asaperssonfab67072017-04-04 05:51:49 -07003230 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003232
asaperssonfab67072017-04-04 05:51:49 -07003233 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003234 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003235
kthelgason876222f2016-11-29 01:44:11 -08003236 // Expect a scale down.
3237 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003238 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003239
asapersson02465b82017-04-10 01:12:52 -07003240 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003241 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003242 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003243 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003244
asaperssonfab67072017-04-04 05:51:49 -07003245 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003246 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003247 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003249
asaperssonfab67072017-04-04 05:51:49 -07003250 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003251 EXPECT_EQ(std::numeric_limits<int>::max(),
3252 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003253
asaperssonfab67072017-04-04 05:51:49 -07003254 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003256 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003257 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003258
asapersson02465b82017-04-10 01:12:52 -07003259 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003260 EXPECT_EQ(std::numeric_limits<int>::max(),
3261 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003262
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003264}
3265
mflodmancc3d4422017-08-03 08:27:51 -07003266TEST_F(VideoStreamEncoderTest,
3267 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003268 const int kWidth = 1280;
3269 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003270 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003271 DataRate::BitsPerSec(kTargetBitrateBps),
3272 DataRate::BitsPerSec(kTargetBitrateBps),
3273 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003274
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003275 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003276 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003277 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003278 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003279
3280 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003281 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003282 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003283 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3284 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3285
3286 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003287 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003288 EXPECT_THAT(source.sink_wants(),
3289 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003290 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3291 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3292 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3293
3294 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003295 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003296 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3297 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3298 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3299
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003301}
3302
mflodmancc3d4422017-08-03 08:27:51 -07003303TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003304 const int kWidth = 1280;
3305 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003306 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003307 DataRate::BitsPerSec(kTargetBitrateBps),
3308 DataRate::BitsPerSec(kTargetBitrateBps),
3309 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003310
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003311 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003312 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003313 video_stream_encoder_->SetSource(&source,
3314 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003315 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3316 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003318
3319 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003321 EXPECT_THAT(source.sink_wants(),
3322 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003323 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3325 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3326
3327 // Trigger adapt down for same input resolution, expect no change.
3328 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3329 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003330 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003331 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3332 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3333 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3334
3335 // Trigger adapt down for larger input resolution, expect no change.
3336 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3337 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003339 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3340 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3341 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3342
mflodmancc3d4422017-08-03 08:27:51 -07003343 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003344}
3345
mflodmancc3d4422017-08-03 08:27:51 -07003346TEST_F(VideoStreamEncoderTest,
3347 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003348 const int kWidth = 1280;
3349 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003350 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003351 DataRate::BitsPerSec(kTargetBitrateBps),
3352 DataRate::BitsPerSec(kTargetBitrateBps),
3353 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003354
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003355 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003356 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003358 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003359
3360 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003361 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003362 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003363 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3364 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3365
3366 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003367 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003368 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003369 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3370 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3371
mflodmancc3d4422017-08-03 08:27:51 -07003372 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003373}
3374
mflodmancc3d4422017-08-03 08:27:51 -07003375TEST_F(VideoStreamEncoderTest,
3376 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003377 const int kWidth = 1280;
3378 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003379 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003380 DataRate::BitsPerSec(kTargetBitrateBps),
3381 DataRate::BitsPerSec(kTargetBitrateBps),
3382 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003383
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003384 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003385 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003386 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003387 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003388
3389 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003390 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003391 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003392 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003393 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3394
3395 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003396 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003397 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003398 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003399 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3400
mflodmancc3d4422017-08-03 08:27:51 -07003401 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003402}
3403
mflodmancc3d4422017-08-03 08:27:51 -07003404TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003405 const int kWidth = 1280;
3406 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003408 DataRate::BitsPerSec(kTargetBitrateBps),
3409 DataRate::BitsPerSec(kTargetBitrateBps),
3410 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003411
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003412 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003413 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003414 video_stream_encoder_->SetSource(&source,
3415 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003416
3417 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3418 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3422 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3423
3424 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003425 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003426 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003427 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3428 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3429 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003432}
3433
mflodmancc3d4422017-08-03 08:27:51 -07003434TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003435 const int kWidth = 1280;
3436 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003438 DataRate::BitsPerSec(kTargetBitrateBps),
3439 DataRate::BitsPerSec(kTargetBitrateBps),
3440 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003441
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003442 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003443 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003444 video_stream_encoder_->SetSource(&source,
3445 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003446
3447 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3448 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003449 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003450 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3451 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3452 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3453
3454 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003455 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003456 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3458 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
mflodmancc3d4422017-08-03 08:27:51 -07003461 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003462}
3463
mflodmancc3d4422017-08-03 08:27:51 -07003464TEST_F(VideoStreamEncoderTest,
3465 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003466 const int kWidth = 1280;
3467 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003468 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003469 DataRate::BitsPerSec(kTargetBitrateBps),
3470 DataRate::BitsPerSec(kTargetBitrateBps),
3471 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003472
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003473 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003474 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003475 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003476 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003477 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003478
3479 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003480 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003481 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003482 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3483 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3484
3485 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003486 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003487 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003488 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003489 EXPECT_THAT(source.sink_wants(),
3490 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3492 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3493
3494 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003495 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003496 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3498 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3499 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3500
mflodmancc3d4422017-08-03 08:27:51 -07003501 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003502}
3503
mflodmancc3d4422017-08-03 08:27:51 -07003504TEST_F(VideoStreamEncoderTest,
3505 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003506 const int kWidth = 1280;
3507 const int kHeight = 720;
3508 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003510 DataRate::BitsPerSec(kTargetBitrateBps),
3511 DataRate::BitsPerSec(kTargetBitrateBps),
3512 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003513
3514 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3515 stats.input_frame_rate = kInputFps;
3516 stats_proxy_->SetMockStats(stats);
3517
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003518 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003519 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3520 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003521 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003522
3523 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003524 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003525 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3526 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003527 EXPECT_THAT(video_source_.sink_wants(),
3528 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003529
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003530 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003531 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003532 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003533 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003534 // Give the encoder queue time to process the change in degradation preference
3535 // by waiting for an encoded frame.
3536 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3537 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003538 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003539
3540 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003541 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003542 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3543 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003544 EXPECT_THAT(new_video_source.sink_wants(),
3545 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003546
3547 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003548 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003549 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003550
mflodmancc3d4422017-08-03 08:27:51 -07003551 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003552}
3553
mflodmancc3d4422017-08-03 08:27:51 -07003554TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003555 const int kWidth = 1280;
3556 const int kHeight = 720;
3557 const size_t kNumFrames = 10;
3558
Henrik Boström381d1092020-05-12 18:49:07 +02003559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003560 DataRate::BitsPerSec(kTargetBitrateBps),
3561 DataRate::BitsPerSec(kTargetBitrateBps),
3562 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003563
asaperssond0de2952017-04-21 01:47:31 -07003564 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003565 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003566 video_source_.set_adaptation_enabled(true);
3567
3568 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3569 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3570
3571 int downscales = 0;
3572 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003573 video_source_.IncomingCapturedFrame(
3574 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3575 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003576
asaperssonfab67072017-04-04 05:51:49 -07003577 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003578 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003579 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003580 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003581
3582 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3583 ++downscales;
3584
3585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_EQ(downscales,
3587 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3588 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003589 }
mflodmancc3d4422017-08-03 08:27:51 -07003590 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003591}
3592
mflodmancc3d4422017-08-03 08:27:51 -07003593TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003594 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3595 const int kWidth = 1280;
3596 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003598 DataRate::BitsPerSec(kTargetBitrateBps),
3599 DataRate::BitsPerSec(kTargetBitrateBps),
3600 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003601
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003602 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003603 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003604 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003605 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003606 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003607
Åsa Persson8c1bf952018-09-13 10:42:19 +02003608 int64_t timestamp_ms = kFrameIntervalMs;
3609 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003610 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003611 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3613 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614
3615 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003616 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003617 timestamp_ms += kFrameIntervalMs;
3618 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3619 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003620 EXPECT_THAT(source.sink_wants(),
3621 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003622 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3624
3625 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003626 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003627 timestamp_ms += kFrameIntervalMs;
3628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003629 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003630 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003631 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3632 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3633
3634 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003635 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003636 timestamp_ms += kFrameIntervalMs;
3637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3638 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003639 EXPECT_THAT(source.sink_wants(),
3640 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003641 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3642 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3643
3644 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003645 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003646 timestamp_ms += kFrameIntervalMs;
3647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003648 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003649 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3651 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3652
mflodmancc3d4422017-08-03 08:27:51 -07003653 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003654}
3655
mflodmancc3d4422017-08-03 08:27:51 -07003656TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003657 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3658 const int kWidth = 1280;
3659 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003661 DataRate::BitsPerSec(kTargetBitrateBps),
3662 DataRate::BitsPerSec(kTargetBitrateBps),
3663 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003664
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003665 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003666 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003667 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003668 video_stream_encoder_->SetSource(&source,
3669 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003670
Åsa Persson8c1bf952018-09-13 10:42:19 +02003671 int64_t timestamp_ms = kFrameIntervalMs;
3672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003673 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003674 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3676 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3677
3678 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003679 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003680 timestamp_ms += kFrameIntervalMs;
3681 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3682 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003683 EXPECT_THAT(source.sink_wants(),
3684 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003685 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3686 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3687
3688 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003689 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003690 timestamp_ms += kFrameIntervalMs;
3691 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003692 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003693 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003694 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3695 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3696
3697 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003698 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003699 timestamp_ms += kFrameIntervalMs;
3700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3701 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003702 EXPECT_THAT(source.sink_wants(),
3703 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003704 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3705 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3706
3707 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003708 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003709 timestamp_ms += kFrameIntervalMs;
3710 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003711 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003712 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003713 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3714 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3715
mflodmancc3d4422017-08-03 08:27:51 -07003716 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003717}
3718
Sergey Silkin41c650b2019-10-14 13:12:19 +02003719TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3720 fake_encoder_.SetResolutionBitrateLimits(
3721 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3722
Henrik Boström381d1092020-05-12 18:49:07 +02003723 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003724 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3725 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3726 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3727 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003728
3729 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003730 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003731 source.set_adaptation_enabled(true);
3732 video_stream_encoder_->SetSource(
3733 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3734
3735 // Insert 720p frame.
3736 int64_t timestamp_ms = kFrameIntervalMs;
3737 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3738 WaitForEncodedFrame(1280, 720);
3739
3740 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003742 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3743 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3744 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3745 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003746 video_stream_encoder_->TriggerQualityLow();
3747
3748 // Insert 720p frame. It should be downscaled and encoded.
3749 timestamp_ms += kFrameIntervalMs;
3750 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3751 WaitForEncodedFrame(960, 540);
3752
3753 // Trigger adapt up. Higher resolution should not be requested duo to lack
3754 // of bitrate.
3755 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003756 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003757
3758 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003759 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003760 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3761 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3762 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3763 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003764
3765 // Trigger adapt up. Higher resolution should be requested.
3766 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003767 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003768
3769 video_stream_encoder_->Stop();
3770}
3771
3772TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3773 fake_encoder_.SetResolutionBitrateLimits(
3774 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3775
3776 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003777 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003778 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3779 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3780 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3781 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003782
3783 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003784 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003785 source.set_adaptation_enabled(true);
3786 video_stream_encoder_->SetSource(
3787 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3788
3789 // Insert 720p frame. It should be dropped and lower resolution should be
3790 // requested.
3791 int64_t timestamp_ms = kFrameIntervalMs;
3792 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3793 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003794 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003795
3796 // Insert 720p frame. It should be downscaled and encoded.
3797 timestamp_ms += kFrameIntervalMs;
3798 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3799 WaitForEncodedFrame(960, 540);
3800
3801 video_stream_encoder_->Stop();
3802}
3803
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003804class BalancedDegradationTest : public VideoStreamEncoderTest {
3805 protected:
3806 void SetupTest() {
3807 // Reset encoder for field trials to take effect.
3808 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003809 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003810
3811 // Enable BALANCED preference.
3812 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003813 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3814 }
3815
3816 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003818 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3819 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003820 }
3821
Åsa Persson45b176f2019-09-30 11:19:05 +02003822 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003823 timestamp_ms_ += kFrameIntervalMs;
3824 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003825 }
3826
3827 void InsertFrameAndWaitForEncoded() {
3828 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003829 sink_.WaitForEncodedFrame(timestamp_ms_);
3830 }
3831
3832 const int kWidth = 640; // pixels:640x360=230400
3833 const int kHeight = 360;
3834 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3835 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003836 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003837};
3838
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003839TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003840 test::ScopedFieldTrials field_trials(
3841 "WebRTC-Video-BalancedDegradationSettings/"
3842 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3843 SetupTest();
3844
3845 // Force input frame rate.
3846 const int kInputFps = 24;
3847 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3848 stats.input_frame_rate = kInputFps;
3849 stats_proxy_->SetMockStats(stats);
3850
Åsa Persson45b176f2019-09-30 11:19:05 +02003851 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003852 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003853
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003854 // Trigger adapt down, expect scaled down framerate and resolution,
3855 // since Fps diff (input-requested:0) < threshold.
3856 video_stream_encoder_->TriggerQualityLow();
3857 EXPECT_THAT(source_.sink_wants(),
3858 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003859
3860 video_stream_encoder_->Stop();
3861}
3862
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003863TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003864 test::ScopedFieldTrials field_trials(
3865 "WebRTC-Video-BalancedDegradationSettings/"
3866 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3867 SetupTest();
3868
3869 // Force input frame rate.
3870 const int kInputFps = 25;
3871 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3872 stats.input_frame_rate = kInputFps;
3873 stats_proxy_->SetMockStats(stats);
3874
Åsa Persson45b176f2019-09-30 11:19:05 +02003875 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003876 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003877
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003878 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3879 // Fps diff (input-requested:1) == threshold.
3880 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003881 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003882
3883 video_stream_encoder_->Stop();
3884}
3885
3886TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3887 test::ScopedFieldTrials field_trials(
3888 "WebRTC-Video-BalancedDegradationSettings/"
3889 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3890 SetupTest();
3891
3892 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3893
Åsa Persson45b176f2019-09-30 11:19:05 +02003894 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003895 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003896
3897 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3898 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003900
3901 video_stream_encoder_->Stop();
3902}
3903
Åsa Perssonccfb3402019-09-25 15:13:04 +02003904TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003905 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003906 "WebRTC-Video-BalancedDegradationSettings/"
3907 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003908 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003909
Åsa Persson1b247f12019-08-14 17:26:39 +02003910 const int kMinBitrateBps = 425000;
3911 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003912 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003913
Åsa Persson45b176f2019-09-30 11:19:05 +02003914 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003915 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003916 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3917
3918 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3919 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003920 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003921 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003922 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3923
3924 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3925 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003926 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003927 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003928 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3929
Åsa Persson30ab0152019-08-27 12:22:33 +02003930 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3931 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003932 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003933 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003934 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003935 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3936
3937 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003938 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003939 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003940 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003941
Åsa Persson30ab0152019-08-27 12:22:33 +02003942 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003943 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003944 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003945 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003946 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003947 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3948
3949 video_stream_encoder_->Stop();
3950}
3951
Åsa Perssonccfb3402019-09-25 15:13:04 +02003952TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003953 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3954 test::ScopedFieldTrials field_trials(
3955 "WebRTC-Video-BalancedDegradationSettings/"
3956 "pixels:57600|129600|230400,fps:7|24|24/");
3957 SetupTest();
3958 OnBitrateUpdated(kLowTargetBitrateBps);
3959
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003960 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003961
3962 // Insert frame, expect scaled down:
3963 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3964 InsertFrame();
3965 EXPECT_FALSE(WaitForFrame(1000));
3966 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3967 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3968
3969 // Insert frame, expect scaled down:
3970 // resolution (320x180@24fps).
3971 InsertFrame();
3972 EXPECT_FALSE(WaitForFrame(1000));
3973 EXPECT_LT(source_.sink_wants().max_pixel_count,
3974 source_.last_wants().max_pixel_count);
3975 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3976
3977 // Frame should not be dropped (min pixels per frame reached).
3978 InsertFrameAndWaitForEncoded();
3979
3980 video_stream_encoder_->Stop();
3981}
3982
3983TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003984 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003985 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003986 "WebRTC-Video-BalancedDegradationSettings/"
3987 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003988 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003989
Åsa Persson30ab0152019-08-27 12:22:33 +02003990 const int kResolutionMinBitrateBps = 435000;
3991 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003992 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003993
Åsa Persson45b176f2019-09-30 11:19:05 +02003994 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003995 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003996 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3997
3998 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3999 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004000 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004001 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004002 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4003
4004 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4005 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004006 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004007 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004008 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4009
4010 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4011 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004012 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004013 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004014 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4015
Åsa Persson30ab0152019-08-27 12:22:33 +02004016 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4017 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004018 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004019 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004020 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4021
4022 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4023 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004024 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004025 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4026
4027 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004028 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004029 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004030 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004031 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004032 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4033
4034 video_stream_encoder_->Stop();
4035}
4036
Åsa Perssonccfb3402019-09-25 15:13:04 +02004037TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004038 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004039 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004040 "WebRTC-Video-BalancedDegradationSettings/"
4041 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004042 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004043
Åsa Persson30ab0152019-08-27 12:22:33 +02004044 const int kMinBitrateBps = 425000;
4045 const int kTooLowMinBitrateBps = 424000;
4046 const int kResolutionMinBitrateBps = 435000;
4047 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004048 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004049
Åsa Persson45b176f2019-09-30 11:19:05 +02004050 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004051 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004052 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4053
4054 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4055 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004056 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004057 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004058 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4059
4060 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4061 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004062 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004063 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004064 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4065
4066 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4067 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004068 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004069 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004070 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4071
4072 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4073 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004074 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004075 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4076
4077 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004078 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004079 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004080 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004081 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004082 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4083
4084 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004085 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004086 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004087 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004088 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4089
4090 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004091 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004092 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004093 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004094 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004095 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4096
Åsa Persson1b247f12019-08-14 17:26:39 +02004097 video_stream_encoder_->Stop();
4098}
4099
mflodmancc3d4422017-08-03 08:27:51 -07004100TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004101 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4102 const int kWidth = 1280;
4103 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004104 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004105 DataRate::BitsPerSec(kTargetBitrateBps),
4106 DataRate::BitsPerSec(kTargetBitrateBps),
4107 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004108
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004109 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004110 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004111 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004112 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004113 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004114
Åsa Persson8c1bf952018-09-13 10:42:19 +02004115 int64_t timestamp_ms = kFrameIntervalMs;
4116 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004117 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004118 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004119 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4120 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4121 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4122 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4123
4124 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004125 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004126 timestamp_ms += kFrameIntervalMs;
4127 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4128 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004129 EXPECT_THAT(source.sink_wants(),
4130 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004131 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4132 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4133 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4134 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4135
4136 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004137 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004138 timestamp_ms += kFrameIntervalMs;
4139 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4140 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004141 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004142 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4143 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4144 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4145 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4146
Jonathan Yubc771b72017-12-08 17:04:29 -08004147 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004148 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004149 timestamp_ms += kFrameIntervalMs;
4150 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4151 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004152 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004153 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4154 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004155 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004156 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4157
Jonathan Yubc771b72017-12-08 17:04:29 -08004158 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004159 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004160 timestamp_ms += kFrameIntervalMs;
4161 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4162 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004163 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004164 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004165 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4166 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4167 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4168 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4169
Jonathan Yubc771b72017-12-08 17:04:29 -08004170 // Trigger quality adapt down, expect no change (min resolution reached).
4171 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004172 timestamp_ms += kFrameIntervalMs;
4173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4174 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004175 EXPECT_THAT(source.sink_wants(), FpsMax());
4176 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004177 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4178 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4179 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4180 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4181
Evan Shrubsole64469032020-06-11 10:45:29 +02004182 // Trigger quality adapt up, expect upscaled resolution (480x270).
4183 video_stream_encoder_->TriggerQualityHigh();
4184 timestamp_ms += kFrameIntervalMs;
4185 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4186 WaitForEncodedFrame(timestamp_ms);
4187 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4188 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4189 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4190 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4191 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4192
4193 // Trigger quality and cpu adapt up since both are most limited, expect
4194 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004195 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004196 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004197 timestamp_ms += kFrameIntervalMs;
4198 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4199 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004200 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004201 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4202 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4203 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004204 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004205
Evan Shrubsole64469032020-06-11 10:45:29 +02004206 // Trigger quality and cpu adapt up since both are most limited, expect
4207 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004208 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004209 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004210 timestamp_ms += kFrameIntervalMs;
4211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4212 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004213 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004214 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004215 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004216 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004217 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4218 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004219
Evan Shrubsole64469032020-06-11 10:45:29 +02004220 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4221 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004222 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004223 timestamp_ms += kFrameIntervalMs;
4224 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4225 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004226 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004227 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4228 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004229 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004230 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004231
4232 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004233 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004234 timestamp_ms += kFrameIntervalMs;
4235 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004236 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004237 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004238 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004239 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4240 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004241 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004242 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004243
mflodmancc3d4422017-08-03 08:27:51 -07004244 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004245}
4246
mflodmancc3d4422017-08-03 08:27:51 -07004247TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004248 const int kWidth = 640;
4249 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004250
Henrik Boström381d1092020-05-12 18:49:07 +02004251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004252 DataRate::BitsPerSec(kTargetBitrateBps),
4253 DataRate::BitsPerSec(kTargetBitrateBps),
4254 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004255
perkj803d97f2016-11-01 11:45:46 -07004256 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004257 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004258 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004259 }
4260
mflodmancc3d4422017-08-03 08:27:51 -07004261 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004262 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004263 video_source_.IncomingCapturedFrame(CreateFrame(
4264 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004265 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004266 }
4267
mflodmancc3d4422017-08-03 08:27:51 -07004268 video_stream_encoder_->Stop();
4269 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004270 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004271
Ying Wangef3998f2019-12-09 13:06:53 +01004272 EXPECT_METRIC_EQ(
4273 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4274 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004275 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4276}
4277
mflodmancc3d4422017-08-03 08:27:51 -07004278TEST_F(VideoStreamEncoderTest,
4279 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004280 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004281 DataRate::BitsPerSec(kTargetBitrateBps),
4282 DataRate::BitsPerSec(kTargetBitrateBps),
4283 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004284 const int kWidth = 640;
4285 const int kHeight = 360;
4286
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004287 video_stream_encoder_->SetSource(&video_source_,
4288 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004289
4290 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4291 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004292 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004293 }
4294
mflodmancc3d4422017-08-03 08:27:51 -07004295 video_stream_encoder_->Stop();
4296 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004297 stats_proxy_.reset();
4298
4299 EXPECT_EQ(0,
4300 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4301}
4302
Per Kjellanderdcef6412020-10-07 15:09:05 +02004303TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4304 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004305 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004306 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004307
4308 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004309 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004310 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004311 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4312 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004313
Henrik Boström381d1092020-05-12 18:49:07 +02004314 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004315 DataRate::BitsPerSec(kLowTargetBitrateBps),
4316 DataRate::BitsPerSec(kLowTargetBitrateBps),
4317 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004318
sprang57c2fff2017-01-16 06:24:02 -08004319 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004320 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4321 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004322 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4323 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4324
Erik Språngd7329ca2019-02-21 21:19:53 +01004325 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004326 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004327 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004328
Per Kjellanderdcef6412020-10-07 15:09:05 +02004329 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004330 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004331 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4332 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004333 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004334 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004335
Per Kjellanderdcef6412020-10-07 15:09:05 +02004336 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004337 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004338 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004339 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004340 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4341 WaitForEncodedFrame(CurrentTimeMs());
4342 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004343 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004344 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004345
mflodmancc3d4422017-08-03 08:27:51 -07004346 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004347}
4348
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004349TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004350 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004351 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004352 kVideoLayersAllocation);
4353
4354 const int kDefaultFps = 30;
4355
4356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4357 DataRate::BitsPerSec(kLowTargetBitrateBps),
4358 DataRate::BitsPerSec(kLowTargetBitrateBps),
4359 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4360
4361 video_source_.IncomingCapturedFrame(
4362 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4363 WaitForEncodedFrame(CurrentTimeMs());
4364 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4365 VideoLayersAllocation last_layer_allocation =
4366 sink_.GetLastVideoLayersAllocation();
4367 // kLowTargetBitrateBps is only enough for one spatial layer.
4368 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4369
4370 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004371 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004372 // Check that encoder has been updated too, not just allocation observer.
4373 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4374 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4375
Erik Språng9d69cbe2020-10-22 17:44:42 +02004376 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004377 int number_of_layers_allocation = 1;
4378 const int64_t start_time_ms = CurrentTimeMs();
4379 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4380 video_source_.IncomingCapturedFrame(
4381 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4382 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004383 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4384 number_of_layers_allocation = sink_.number_of_layers_allocations();
4385 VideoLayersAllocation new_allocation =
4386 sink_.GetLastVideoLayersAllocation();
4387 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4388 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4389 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4390 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4391 .target_bitrate_per_temporal_layer,
4392 last_layer_allocation.active_spatial_layers[0]
4393 .target_bitrate_per_temporal_layer);
4394 last_layer_allocation = new_allocation;
4395 }
4396 }
4397 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4398 video_stream_encoder_->Stop();
4399}
4400
4401TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004402 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004403 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4404 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4405 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004406 VideoEncoderConfig video_encoder_config;
4407 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4408 /* num_streams*/ 3, &video_encoder_config);
4409 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4410 video_encoder_config.content_type =
4411 VideoEncoderConfig::ContentType::kRealtimeVideo;
4412 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004413 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004414 VideoEncoder::GetDefaultVp8Settings());
4415 for (auto& layer : video_encoder_config.simulcast_layers) {
4416 layer.num_temporal_layers = 2;
4417 }
4418 // Simulcast layers are used for enabling/disabling streams.
4419 video_encoder_config.simulcast_layers[0].active = true;
4420 video_encoder_config.simulcast_layers[1].active = false;
4421 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004422 ConfigureEncoder(std::move(video_encoder_config),
4423 VideoStreamEncoder::BitrateAllocationCallbackType::
4424 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004425
4426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4427 DataRate::BitsPerSec(kTargetBitrateBps),
4428 DataRate::BitsPerSec(kTargetBitrateBps),
4429 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4430
4431 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4432 WaitForEncodedFrame(CurrentTimeMs());
4433 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4434 VideoLayersAllocation last_layer_allocation =
4435 sink_.GetLastVideoLayersAllocation();
4436
4437 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4438 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4439 .target_bitrate_per_temporal_layer,
4440 SizeIs(2));
4441 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4442 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4443 video_stream_encoder_->Stop();
4444}
4445
4446TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004447 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004448 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4449 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4450 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004451 VideoEncoderConfig video_encoder_config;
4452 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4453 /* num_streams*/ 3, &video_encoder_config);
4454 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4455 video_encoder_config.content_type =
4456 VideoEncoderConfig::ContentType::kRealtimeVideo;
4457 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004458 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004459 VideoEncoder::GetDefaultVp8Settings());
4460 for (auto& layer : video_encoder_config.simulcast_layers) {
4461 layer.num_temporal_layers = 2;
4462 }
4463 // Simulcast layers are used for enabling/disabling streams.
4464 video_encoder_config.simulcast_layers[0].active = true;
4465 video_encoder_config.simulcast_layers[1].active = false;
4466 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004467 ConfigureEncoder(std::move(video_encoder_config),
4468 VideoStreamEncoder::BitrateAllocationCallbackType::
4469 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004470
4471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4472 DataRate::BitsPerSec(kTargetBitrateBps),
4473 DataRate::BitsPerSec(kTargetBitrateBps),
4474 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4475
4476 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4477 WaitForEncodedFrame(CurrentTimeMs());
4478 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4479 VideoLayersAllocation last_layer_allocation =
4480 sink_.GetLastVideoLayersAllocation();
4481
4482 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4483 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4484 .target_bitrate_per_temporal_layer,
4485 SizeIs(2));
4486 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4487
4488 video_stream_encoder_->Stop();
4489}
4490
4491TEST_F(VideoStreamEncoderTest,
4492 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4493 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4494 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004495 VideoEncoderConfig video_encoder_config;
4496 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4497 /* num_streams*/ 1, &video_encoder_config);
4498 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4499 video_encoder_config.content_type =
4500 VideoEncoderConfig::ContentType::kRealtimeVideo;
4501 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4502 vp9_settings.numberOfSpatialLayers = 2;
4503 vp9_settings.numberOfTemporalLayers = 2;
4504 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4505 vp9_settings.automaticResizeOn = false;
4506 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004507 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004508 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004509 ConfigureEncoder(std::move(video_encoder_config),
4510 VideoStreamEncoder::BitrateAllocationCallbackType::
4511 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004512
4513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4514 DataRate::BitsPerSec(kTargetBitrateBps),
4515 DataRate::BitsPerSec(kTargetBitrateBps),
4516 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4517
4518 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4519 WaitForEncodedFrame(CurrentTimeMs());
4520 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4521 VideoLayersAllocation last_layer_allocation =
4522 sink_.GetLastVideoLayersAllocation();
4523
4524 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4525 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4526 .target_bitrate_per_temporal_layer,
4527 SizeIs(2));
4528 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4529 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4530 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4531 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4532 .target_bitrate_per_temporal_layer,
4533 SizeIs(2));
4534 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4535 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4536 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4537
4538 // Since full SVC is used, expect the top layer to utilize the full target
4539 // rate.
4540 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4541 .target_bitrate_per_temporal_layer[1],
4542 DataRate::BitsPerSec(kTargetBitrateBps));
4543 video_stream_encoder_->Stop();
4544}
4545
4546TEST_F(VideoStreamEncoderTest,
4547 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4548 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4549 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004550 VideoEncoderConfig video_encoder_config;
4551 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4552 /* num_streams*/ 1, &video_encoder_config);
4553 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4554 video_encoder_config.content_type =
4555 VideoEncoderConfig::ContentType::kRealtimeVideo;
4556 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4557 vp9_settings.numberOfSpatialLayers = 2;
4558 vp9_settings.numberOfTemporalLayers = 2;
4559 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4560 vp9_settings.automaticResizeOn = false;
4561 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004562 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004563 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004564 ConfigureEncoder(std::move(video_encoder_config),
4565 VideoStreamEncoder::BitrateAllocationCallbackType::
4566 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004567
4568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4569 DataRate::BitsPerSec(kTargetBitrateBps),
4570 DataRate::BitsPerSec(kTargetBitrateBps),
4571 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4572
4573 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4574 WaitForEncodedFrame(CurrentTimeMs());
4575 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4576 VideoLayersAllocation last_layer_allocation =
4577 sink_.GetLastVideoLayersAllocation();
4578
4579 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4580 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4581 .target_bitrate_per_temporal_layer,
4582 SizeIs(1));
4583 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4584 .target_bitrate_per_temporal_layer,
4585 SizeIs(1));
4586 // Since full SVC is used, expect the top layer to utilize the full target
4587 // rate.
4588 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4589 .target_bitrate_per_temporal_layer[0],
4590 DataRate::BitsPerSec(kTargetBitrateBps));
4591 video_stream_encoder_->Stop();
4592}
4593
4594TEST_F(VideoStreamEncoderTest,
4595 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4596 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4597 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004598 VideoEncoderConfig video_encoder_config;
4599 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4600 /* num_streams*/ 1, &video_encoder_config);
4601 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4602 video_encoder_config.content_type =
4603 VideoEncoderConfig::ContentType::kRealtimeVideo;
4604 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4605 vp9_settings.numberOfSpatialLayers = 2;
4606 vp9_settings.numberOfTemporalLayers = 2;
4607 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4608 vp9_settings.automaticResizeOn = false;
4609 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004610 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004611 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004612 ConfigureEncoder(std::move(video_encoder_config),
4613 VideoStreamEncoder::BitrateAllocationCallbackType::
4614 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004615
4616 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4617 DataRate::BitsPerSec(kTargetBitrateBps),
4618 DataRate::BitsPerSec(kTargetBitrateBps),
4619 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4620
4621 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4622 WaitForEncodedFrame(CurrentTimeMs());
4623 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4624 VideoLayersAllocation last_layer_allocation =
4625 sink_.GetLastVideoLayersAllocation();
4626
4627 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4628 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4629 .target_bitrate_per_temporal_layer,
4630 SizeIs(2));
4631 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4632 .target_bitrate_per_temporal_layer,
4633 SizeIs(2));
4634 // Since KSVC is, spatial layers are independend except on key frames.
4635 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4636 .target_bitrate_per_temporal_layer[1],
4637 DataRate::BitsPerSec(kTargetBitrateBps));
4638 video_stream_encoder_->Stop();
4639}
4640
4641TEST_F(VideoStreamEncoderTest,
4642 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4643 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4644 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4645 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004646 VideoEncoderConfig video_encoder_config;
4647 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4648 /* num_streams*/ 1, &video_encoder_config);
4649 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4650 video_encoder_config.content_type =
4651 VideoEncoderConfig::ContentType::kRealtimeVideo;
4652 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4653 vp9_settings.numberOfSpatialLayers = 3;
4654 vp9_settings.numberOfTemporalLayers = 2;
4655 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4656 vp9_settings.automaticResizeOn = false;
4657 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004658 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004659 vp9_settings);
4660 // Simulcast layers are used for enabling/disabling streams.
4661 video_encoder_config.simulcast_layers.resize(3);
4662 video_encoder_config.simulcast_layers[0].active = false;
4663 video_encoder_config.simulcast_layers[1].active = true;
4664 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004665 ConfigureEncoder(std::move(video_encoder_config),
4666 VideoStreamEncoder::BitrateAllocationCallbackType::
4667 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004668
4669 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4670 DataRate::BitsPerSec(kTargetBitrateBps),
4671 DataRate::BitsPerSec(kTargetBitrateBps),
4672 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4673
4674 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4675 WaitForEncodedFrame(CurrentTimeMs());
4676 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4677 VideoLayersAllocation last_layer_allocation =
4678 sink_.GetLastVideoLayersAllocation();
4679
4680 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4681 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4682 .target_bitrate_per_temporal_layer,
4683 SizeIs(2));
4684 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4685 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4686
4687 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4688 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4689 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4690 .target_bitrate_per_temporal_layer,
4691 SizeIs(2));
4692 // Since full SVC is used, expect the top layer to utilize the full target
4693 // rate.
4694 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4695 .target_bitrate_per_temporal_layer[1],
4696 DataRate::BitsPerSec(kTargetBitrateBps));
4697 video_stream_encoder_->Stop();
4698}
4699
4700TEST_F(VideoStreamEncoderTest,
4701 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4702 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4703 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4704 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004705 VideoEncoderConfig video_encoder_config;
4706 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4707 /* num_streams*/ 1, &video_encoder_config);
4708 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4709 video_encoder_config.content_type =
4710 VideoEncoderConfig::ContentType::kRealtimeVideo;
4711 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4712 vp9_settings.numberOfSpatialLayers = 3;
4713 vp9_settings.numberOfTemporalLayers = 2;
4714 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4715 vp9_settings.automaticResizeOn = false;
4716 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004717 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 vp9_settings);
4719 // Simulcast layers are used for enabling/disabling streams.
4720 video_encoder_config.simulcast_layers.resize(3);
4721 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004722 ConfigureEncoder(std::move(video_encoder_config),
4723 VideoStreamEncoder::BitrateAllocationCallbackType::
4724 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725
4726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4727 DataRate::BitsPerSec(kTargetBitrateBps),
4728 DataRate::BitsPerSec(kTargetBitrateBps),
4729 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4730
4731 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4732 WaitForEncodedFrame(CurrentTimeMs());
4733 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4734 VideoLayersAllocation last_layer_allocation =
4735 sink_.GetLastVideoLayersAllocation();
4736
4737 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4738 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4739 .target_bitrate_per_temporal_layer,
4740 SizeIs(2));
4741 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4742 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4743
4744 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4745 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4746 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4747 .target_bitrate_per_temporal_layer,
4748 SizeIs(2));
4749 video_stream_encoder_->Stop();
4750}
4751
4752TEST_F(VideoStreamEncoderTest,
4753 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4754 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4755 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4756 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004757 VideoEncoderConfig video_encoder_config;
4758 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4759 /* num_streams*/ 1, &video_encoder_config);
4760 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4761 video_encoder_config.content_type =
4762 VideoEncoderConfig::ContentType::kRealtimeVideo;
4763 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4764 vp9_settings.numberOfSpatialLayers = 3;
4765 vp9_settings.numberOfTemporalLayers = 2;
4766 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4767 vp9_settings.automaticResizeOn = false;
4768 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004769 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004770 vp9_settings);
4771 // Simulcast layers are used for enabling/disabling streams.
4772 video_encoder_config.simulcast_layers.resize(3);
4773 video_encoder_config.simulcast_layers[0].active = false;
4774 video_encoder_config.simulcast_layers[1].active = false;
4775 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004776 ConfigureEncoder(std::move(video_encoder_config),
4777 VideoStreamEncoder::BitrateAllocationCallbackType::
4778 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004779
4780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4781 DataRate::BitsPerSec(kTargetBitrateBps),
4782 DataRate::BitsPerSec(kTargetBitrateBps),
4783 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4784
4785 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4786 WaitForEncodedFrame(CurrentTimeMs());
4787 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4788 VideoLayersAllocation last_layer_allocation =
4789 sink_.GetLastVideoLayersAllocation();
4790
4791 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4792 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4793 .target_bitrate_per_temporal_layer,
4794 SizeIs(2));
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4796 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4797 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4798 .target_bitrate_per_temporal_layer[1],
4799 DataRate::BitsPerSec(kTargetBitrateBps));
4800 video_stream_encoder_->Stop();
4801}
4802
4803TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4804 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004805 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004806 kVideoLayersAllocation);
4807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4808 DataRate::BitsPerSec(kTargetBitrateBps),
4809 DataRate::BitsPerSec(kTargetBitrateBps),
4810 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4811
4812 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4813 WaitForEncodedFrame(CurrentTimeMs());
4814 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4815 VideoLayersAllocation last_layer_allocation =
4816 sink_.GetLastVideoLayersAllocation();
4817
4818 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4819 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4820 .target_bitrate_per_temporal_layer,
4821 SizeIs(1));
4822 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4823 .target_bitrate_per_temporal_layer[0],
4824 DataRate::BitsPerSec(kTargetBitrateBps));
4825 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4826 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4827 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4828 video_stream_encoder_->Stop();
4829}
4830
4831TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004832 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4833 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004834 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004835 kVideoLayersAllocation);
4836
4837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4838 DataRate::BitsPerSec(kLowTargetBitrateBps),
4839 DataRate::BitsPerSec(kLowTargetBitrateBps),
4840 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4841
4842 video_source_.IncomingCapturedFrame(
4843 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4844 WaitForEncodedFrame(CurrentTimeMs());
4845 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4846 VideoLayersAllocation last_layer_allocation =
4847 sink_.GetLastVideoLayersAllocation();
4848 // kLowTargetBitrateBps is only enough for one spatial layer.
4849 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4850 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4851 .target_bitrate_per_temporal_layer[0],
4852 DataRate::BitsPerSec(kLowTargetBitrateBps));
4853
4854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4855 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4856 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4857 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4858 video_source_.IncomingCapturedFrame(
4859 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4860 WaitForEncodedFrame(CurrentTimeMs());
4861
4862 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4863 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4864 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4865 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4866 .target_bitrate_per_temporal_layer[0],
4867 DataRate::Zero());
4868
4869 video_stream_encoder_->Stop();
4870}
4871
Per Kjellander4190ce92020-12-15 17:24:55 +01004872TEST_F(VideoStreamEncoderTest,
4873 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4874 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004875 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004876 kVideoLayersAllocation);
4877
4878 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4879 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4880 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4881 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4882
4883 video_source_.IncomingCapturedFrame(
4884 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4885 WaitForEncodedFrame(CurrentTimeMs());
4886 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4887 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4888 SizeIs(2));
4889 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4890 codec_width_);
4891 EXPECT_EQ(
4892 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4893 codec_height_);
4894
4895 video_source_.IncomingCapturedFrame(
4896 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4897 WaitForEncodedFrame(CurrentTimeMs());
4898 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4899 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4900 SizeIs(2));
4901 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4902 codec_width_ / 2);
4903 EXPECT_EQ(
4904 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4905 codec_height_ / 2);
4906
4907 video_stream_encoder_->Stop();
4908}
4909
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004910TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4911 // 2 TLs configured, temporal layers supported by encoder.
4912 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004913 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004914 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004915 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004916 fake_encoder_.SetTemporalLayersSupported(0, true);
4917
4918 // Bitrate allocated across temporal layers.
4919 const int kTl0Bps = kTargetBitrateBps *
4920 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004921 kNumTemporalLayers, /*temporal_id*/ 0,
4922 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004923 const int kTl1Bps = kTargetBitrateBps *
4924 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004925 kNumTemporalLayers, /*temporal_id*/ 1,
4926 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004927 VideoBitrateAllocation expected_bitrate;
4928 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4929 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4930
4931 VerifyAllocatedBitrate(expected_bitrate);
4932 video_stream_encoder_->Stop();
4933}
4934
4935TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4936 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004937 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004938 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004939 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004940 fake_encoder_.SetTemporalLayersSupported(0, false);
4941
4942 // Temporal layers not supported by the encoder.
4943 // Total bitrate should be at ti:0.
4944 VideoBitrateAllocation expected_bitrate;
4945 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4946
4947 VerifyAllocatedBitrate(expected_bitrate);
4948 video_stream_encoder_->Stop();
4949}
4950
4951TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004952 webrtc::test::ScopedFieldTrials field_trials(
4953 "WebRTC-Video-QualityScalerSettings/"
4954 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4955 // Reset encoder for field trials to take effect.
4956 ConfigureEncoder(video_encoder_config_.Copy());
4957
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004958 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004959 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004960 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004961 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004962 fake_encoder_.SetTemporalLayersSupported(0, true);
4963 fake_encoder_.SetTemporalLayersSupported(1, false);
4964
4965 const int kS0Bps = 150000;
4966 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004967 kS0Bps *
4968 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4969 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004970 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004971 kS0Bps *
4972 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4973 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004974 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4975 // Temporal layers not supported by si:1.
4976 VideoBitrateAllocation expected_bitrate;
4977 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4978 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4979 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4980
4981 VerifyAllocatedBitrate(expected_bitrate);
4982 video_stream_encoder_->Stop();
4983}
4984
Niels Möller7dc26b72017-12-06 10:27:48 +01004985TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4986 const int kFrameWidth = 1280;
4987 const int kFrameHeight = 720;
4988 const int kFramerate = 24;
4989
Henrik Boström381d1092020-05-12 18:49:07 +02004990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004991 DataRate::BitsPerSec(kTargetBitrateBps),
4992 DataRate::BitsPerSec(kTargetBitrateBps),
4993 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004994 test::FrameForwarder source;
4995 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004996 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004997
4998 // Insert a single frame, triggering initial configuration.
4999 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5000 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5001
5002 EXPECT_EQ(
5003 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5004 kDefaultFramerate);
5005
5006 // Trigger reconfigure encoder (without resetting the entire instance).
5007 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005008 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5009 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005010 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005011 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005012 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005013 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5014
5015 // Detector should be updated with fps limit from codec config.
5016 EXPECT_EQ(
5017 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5018 kFramerate);
5019
5020 // Trigger overuse, max framerate should be reduced.
5021 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5022 stats.input_frame_rate = kFramerate;
5023 stats_proxy_->SetMockStats(stats);
5024 video_stream_encoder_->TriggerCpuOveruse();
5025 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5026 int adapted_framerate =
5027 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5028 EXPECT_LT(adapted_framerate, kFramerate);
5029
5030 // Trigger underuse, max framerate should go back to codec configured fps.
5031 // Set extra low fps, to make sure it's actually reset, not just incremented.
5032 stats = stats_proxy_->GetStats();
5033 stats.input_frame_rate = adapted_framerate / 2;
5034 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005035 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005036 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5037 EXPECT_EQ(
5038 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5039 kFramerate);
5040
5041 video_stream_encoder_->Stop();
5042}
5043
5044TEST_F(VideoStreamEncoderTest,
5045 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5046 const int kFrameWidth = 1280;
5047 const int kFrameHeight = 720;
5048 const int kLowFramerate = 15;
5049 const int kHighFramerate = 25;
5050
Henrik Boström381d1092020-05-12 18:49:07 +02005051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005052 DataRate::BitsPerSec(kTargetBitrateBps),
5053 DataRate::BitsPerSec(kTargetBitrateBps),
5054 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005055 test::FrameForwarder source;
5056 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005057 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005058
5059 // Trigger initial configuration.
5060 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005061 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5062 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005063 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005064 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005065 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005066 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005067 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5068
5069 EXPECT_EQ(
5070 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5071 kLowFramerate);
5072
5073 // Trigger overuse, max framerate should be reduced.
5074 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5075 stats.input_frame_rate = kLowFramerate;
5076 stats_proxy_->SetMockStats(stats);
5077 video_stream_encoder_->TriggerCpuOveruse();
5078 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5079 int adapted_framerate =
5080 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5081 EXPECT_LT(adapted_framerate, kLowFramerate);
5082
5083 // Reconfigure the encoder with a new (higher max framerate), max fps should
5084 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005085 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005086 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5087 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005088 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005089 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5090
5091 EXPECT_EQ(
5092 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5093 adapted_framerate);
5094
5095 // Trigger underuse, max framerate should go back to codec configured fps.
5096 stats = stats_proxy_->GetStats();
5097 stats.input_frame_rate = adapted_framerate;
5098 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005099 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005100 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5101 EXPECT_EQ(
5102 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5103 kHighFramerate);
5104
5105 video_stream_encoder_->Stop();
5106}
5107
mflodmancc3d4422017-08-03 08:27:51 -07005108TEST_F(VideoStreamEncoderTest,
5109 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005110 const int kFrameWidth = 1280;
5111 const int kFrameHeight = 720;
5112 const int kFramerate = 24;
5113
Henrik Boström381d1092020-05-12 18:49:07 +02005114 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005115 DataRate::BitsPerSec(kTargetBitrateBps),
5116 DataRate::BitsPerSec(kTargetBitrateBps),
5117 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005118 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005119 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005120 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005121
5122 // Trigger initial configuration.
5123 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005124 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5125 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005126 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005127 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005128 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005129 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005130 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005131
Niels Möller7dc26b72017-12-06 10:27:48 +01005132 EXPECT_EQ(
5133 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5134 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005135
5136 // Trigger overuse, max framerate should be reduced.
5137 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5138 stats.input_frame_rate = kFramerate;
5139 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005140 video_stream_encoder_->TriggerCpuOveruse();
5141 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005142 int adapted_framerate =
5143 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005144 EXPECT_LT(adapted_framerate, kFramerate);
5145
5146 // Change degradation preference to not enable framerate scaling. Target
5147 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005148 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005149 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005150 EXPECT_EQ(
5151 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5152 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005153
mflodmancc3d4422017-08-03 08:27:51 -07005154 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005155}
5156
mflodmancc3d4422017-08-03 08:27:51 -07005157TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005158 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005160 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5161 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5162 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005163 const int kWidth = 640;
5164 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005165
asaperssonfab67072017-04-04 05:51:49 -07005166 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005167
5168 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005169 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005170
5171 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005172 EXPECT_TRUE_WAIT(
5173 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005174
sprangc5d62e22017-04-02 23:53:04 -07005175 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005176
asaperssonfab67072017-04-04 05:51:49 -07005177 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005178 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005179 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005180
5181 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005182 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005183
Henrik Boström2671dac2020-05-19 16:29:09 +02005184 EXPECT_TRUE_WAIT(
5185 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005186
mflodmancc3d4422017-08-03 08:27:51 -07005187 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005188}
5189
mflodmancc3d4422017-08-03 08:27:51 -07005190TEST_F(VideoStreamEncoderTest,
5191 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005192 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005194 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5195 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5196 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005197 const int kWidth = 640;
5198 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005199
5200 // We expect the n initial frames to get dropped.
5201 int i;
5202 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005203 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005204 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005205 }
5206 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005207 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005208 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005209
5210 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005211 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005212
mflodmancc3d4422017-08-03 08:27:51 -07005213 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005214}
5215
mflodmancc3d4422017-08-03 08:27:51 -07005216TEST_F(VideoStreamEncoderTest,
5217 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005218 const int kWidth = 640;
5219 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005221 DataRate::BitsPerSec(kLowTargetBitrateBps),
5222 DataRate::BitsPerSec(kLowTargetBitrateBps),
5223 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005224
5225 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005226 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005227 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005228
asaperssonfab67072017-04-04 05:51:49 -07005229 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005230 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005231 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005232
mflodmancc3d4422017-08-03 08:27:51 -07005233 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005234}
5235
mflodmancc3d4422017-08-03 08:27:51 -07005236TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005237 const int kWidth = 640;
5238 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005239 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005240
5241 VideoEncoderConfig video_encoder_config;
5242 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5243 // Make format different, to force recreation of encoder.
5244 video_encoder_config.video_format.parameters["foo"] = "foo";
5245 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005246 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005247 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005248 DataRate::BitsPerSec(kLowTargetBitrateBps),
5249 DataRate::BitsPerSec(kLowTargetBitrateBps),
5250 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005251
kthelgasonb83797b2017-02-14 11:57:25 -08005252 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005253 video_stream_encoder_->SetSource(&video_source_,
5254 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005255
asaperssonfab67072017-04-04 05:51:49 -07005256 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005257 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005258 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005259
mflodmancc3d4422017-08-03 08:27:51 -07005260 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005261 fake_encoder_.SetQualityScaling(true);
5262}
5263
Åsa Persson139f4dc2019-08-02 09:29:58 +02005264TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5265 webrtc::test::ScopedFieldTrials field_trials(
5266 "WebRTC-Video-QualityScalerSettings/"
5267 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5268 // Reset encoder for field trials to take effect.
5269 ConfigureEncoder(video_encoder_config_.Copy());
5270 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5271 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5272 const int kWidth = 640;
5273 const int kHeight = 360;
5274
Henrik Boström381d1092020-05-12 18:49:07 +02005275 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005276 DataRate::BitsPerSec(kTargetBitrateBps),
5277 DataRate::BitsPerSec(kTargetBitrateBps),
5278 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005279 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5280 // Frame should not be dropped.
5281 WaitForEncodedFrame(1);
5282
Henrik Boström381d1092020-05-12 18:49:07 +02005283 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005284 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5285 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5286 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005287 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5288 // Frame should not be dropped.
5289 WaitForEncodedFrame(2);
5290
Henrik Boström381d1092020-05-12 18:49:07 +02005291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005292 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5293 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5294 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005295 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5296 // Expect to drop this frame, the wait should time out.
5297 ExpectDroppedFrame();
5298
5299 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005300 EXPECT_TRUE_WAIT(
5301 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005302 video_stream_encoder_->Stop();
5303}
5304
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005305TEST_F(VideoStreamEncoderTest,
5306 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5307 webrtc::test::ScopedFieldTrials field_trials(
5308 "WebRTC-Video-QualityScalerSettings/"
5309 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5310 fake_encoder_.SetQualityScaling(false);
5311 ConfigureEncoder(video_encoder_config_.Copy());
5312 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5313 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5314 const int kWidth = 640;
5315 const int kHeight = 360;
5316
5317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5318 DataRate::BitsPerSec(kTargetBitrateBps),
5319 DataRate::BitsPerSec(kTargetBitrateBps),
5320 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5321 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5322 // Frame should not be dropped.
5323 WaitForEncodedFrame(1);
5324
5325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5326 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5327 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5328 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5329 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5330 // Frame should not be dropped.
5331 WaitForEncodedFrame(2);
5332
5333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5334 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5335 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5336 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5337 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5338 // Not dropped since quality scaling is disabled.
5339 WaitForEncodedFrame(3);
5340
5341 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005342 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005343 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5344
5345 video_stream_encoder_->Stop();
5346}
5347
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005348TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5349 const int kLowTargetBitrateBps = 400000;
5350 // Set simulcast.
5351 ResetEncoder("VP8", 3, 1, 1, false);
5352 fake_encoder_.SetQualityScaling(true);
5353 const int kWidth = 1280;
5354 const int kHeight = 720;
5355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5356 DataRate::BitsPerSec(kLowTargetBitrateBps),
5357 DataRate::BitsPerSec(kLowTargetBitrateBps),
5358 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5359 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5360 // Frame should not be dropped.
5361 WaitForEncodedFrame(1);
5362
5363 // Trigger QVGA "singlecast"
5364 // Update the config.
5365 VideoEncoderConfig video_encoder_config;
5366 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5367 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005368 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005369 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005370 "VP8", /*max qp*/ 56, /*screencast*/ false,
5371 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005372 for (auto& layer : video_encoder_config.simulcast_layers) {
5373 layer.num_temporal_layers = 1;
5374 layer.max_framerate = kDefaultFramerate;
5375 }
5376 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5377 video_encoder_config.content_type =
5378 VideoEncoderConfig::ContentType::kRealtimeVideo;
5379
5380 video_encoder_config.simulcast_layers[0].active = true;
5381 video_encoder_config.simulcast_layers[1].active = false;
5382 video_encoder_config.simulcast_layers[2].active = false;
5383
5384 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5385 kMaxPayloadLength);
5386 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5387
5388 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5389 // Frame should not be dropped.
5390 WaitForEncodedFrame(2);
5391
5392 // Trigger HD "singlecast"
5393 video_encoder_config.simulcast_layers[0].active = false;
5394 video_encoder_config.simulcast_layers[1].active = false;
5395 video_encoder_config.simulcast_layers[2].active = true;
5396
5397 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5398 kMaxPayloadLength);
5399 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5400
5401 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5402 // Frame should be dropped because of initial frame drop.
5403 ExpectDroppedFrame();
5404
5405 // Expect the sink_wants to specify a scaled frame.
5406 EXPECT_TRUE_WAIT(
5407 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5408 video_stream_encoder_->Stop();
5409}
5410
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005411TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5412 const int kLowTargetBitrateBps = 400000;
5413 // Set simulcast.
5414 ResetEncoder("VP9", 1, 1, 3, false);
5415 fake_encoder_.SetQualityScaling(true);
5416 const int kWidth = 1280;
5417 const int kHeight = 720;
5418 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5419 DataRate::BitsPerSec(kLowTargetBitrateBps),
5420 DataRate::BitsPerSec(kLowTargetBitrateBps),
5421 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5422 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5423 // Frame should not be dropped.
5424 WaitForEncodedFrame(1);
5425
5426 // Trigger QVGA "singlecast"
5427 // Update the config.
5428 VideoEncoderConfig video_encoder_config;
5429 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5430 &video_encoder_config);
5431 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5432 vp9_settings.numberOfSpatialLayers = 3;
5433 // Since only one layer is active - automatic resize should be enabled.
5434 vp9_settings.automaticResizeOn = true;
5435 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005436 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005437 vp9_settings);
5438 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5439 video_encoder_config.content_type =
5440 VideoEncoderConfig::ContentType::kRealtimeVideo;
5441 // Currently simulcast layers |active| flags are used to inidicate
5442 // which SVC layers are active.
5443 video_encoder_config.simulcast_layers.resize(3);
5444
5445 video_encoder_config.simulcast_layers[0].active = true;
5446 video_encoder_config.simulcast_layers[1].active = false;
5447 video_encoder_config.simulcast_layers[2].active = false;
5448
5449 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5450 kMaxPayloadLength);
5451 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5452
5453 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5454 // Frame should not be dropped.
5455 WaitForEncodedFrame(2);
5456
5457 // Trigger HD "singlecast"
5458 video_encoder_config.simulcast_layers[0].active = false;
5459 video_encoder_config.simulcast_layers[1].active = false;
5460 video_encoder_config.simulcast_layers[2].active = true;
5461
5462 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5463 kMaxPayloadLength);
5464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5465
5466 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5467 // Frame should be dropped because of initial frame drop.
5468 ExpectDroppedFrame();
5469
5470 // Expect the sink_wants to specify a scaled frame.
5471 EXPECT_TRUE_WAIT(
5472 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5473 video_stream_encoder_->Stop();
5474}
5475
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005476TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005477 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5478 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5479 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5480 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5481 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5482 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5483 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5484 fake_encoder_.SetResolutionBitrateLimits(
5485 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5486
5487 VideoEncoderConfig video_encoder_config;
5488 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5489 &video_encoder_config);
5490 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5491 vp9_settings.numberOfSpatialLayers = 3;
5492 // Since only one layer is active - automatic resize should be enabled.
5493 vp9_settings.automaticResizeOn = true;
5494 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005495 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005496 vp9_settings);
5497 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5498 video_encoder_config.content_type =
5499 VideoEncoderConfig::ContentType::kRealtimeVideo;
5500 // Simulcast layers are used to indicate which spatial layers are active.
5501 video_encoder_config.simulcast_layers.resize(3);
5502 video_encoder_config.simulcast_layers[0].active = false;
5503 video_encoder_config.simulcast_layers[1].active = true;
5504 video_encoder_config.simulcast_layers[2].active = false;
5505
5506 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5507 kMaxPayloadLength);
5508 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5509
5510 // The encoder bitrate limits for 360p should be used.
5511 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5512 EXPECT_FALSE(WaitForFrame(1000));
5513 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5514 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5515 VideoCodecType::kVideoCodecVP9);
5516 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5517 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5518 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5519 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5520 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5521 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5522 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5523 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5524
5525 // The encoder bitrate limits for 270p should be used.
5526 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5527 EXPECT_FALSE(WaitForFrame(1000));
5528 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5529 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5530 VideoCodecType::kVideoCodecVP9);
5531 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5532 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5533 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5534 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5535 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5536 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5537 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5538 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5539
5540 video_stream_encoder_->Stop();
5541}
5542
5543TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005544 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5545 VideoEncoderConfig video_encoder_config;
5546 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5547 &video_encoder_config);
5548 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5549 vp9_settings.numberOfSpatialLayers = 3;
5550 // Since only one layer is active - automatic resize should be enabled.
5551 vp9_settings.automaticResizeOn = true;
5552 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005553 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005554 vp9_settings);
5555 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5556 video_encoder_config.content_type =
5557 VideoEncoderConfig::ContentType::kRealtimeVideo;
5558 // Simulcast layers are used to indicate which spatial layers are active.
5559 video_encoder_config.simulcast_layers.resize(3);
5560 video_encoder_config.simulcast_layers[0].active = false;
5561 video_encoder_config.simulcast_layers[1].active = true;
5562 video_encoder_config.simulcast_layers[2].active = false;
5563
5564 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5565 kMaxPayloadLength);
5566 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5567
5568 // The default bitrate limits for 360p should be used.
5569 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005570 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5571 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005572 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5573 EXPECT_FALSE(WaitForFrame(1000));
5574 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5575 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5576 VideoCodecType::kVideoCodecVP9);
5577 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5578 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5579 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5580 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5581 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5582 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5583 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5584 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5585
5586 // The default bitrate limits for 270p should be used.
5587 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005588 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5589 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005590 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5591 EXPECT_FALSE(WaitForFrame(1000));
5592 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5593 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5594 VideoCodecType::kVideoCodecVP9);
5595 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5596 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5597 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5598 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5599 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5600 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5601 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5602 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5603
5604 video_stream_encoder_->Stop();
5605}
5606
5607TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5608 webrtc::test::ScopedFieldTrials field_trials(
5609 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5610 VideoEncoderConfig video_encoder_config;
5611 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5612 &video_encoder_config);
5613 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5614 vp9_settings.numberOfSpatialLayers = 3;
5615 // Since only one layer is active - automatic resize should be enabled.
5616 vp9_settings.automaticResizeOn = true;
5617 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005618 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005619 vp9_settings);
5620 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5621 video_encoder_config.content_type =
5622 VideoEncoderConfig::ContentType::kRealtimeVideo;
5623 // Simulcast layers are used to indicate which spatial layers are active.
5624 video_encoder_config.simulcast_layers.resize(3);
5625 video_encoder_config.simulcast_layers[0].active = false;
5626 video_encoder_config.simulcast_layers[1].active = true;
5627 video_encoder_config.simulcast_layers[2].active = false;
5628
5629 // Reset encoder for field trials to take effect.
5630 ConfigureEncoder(video_encoder_config.Copy());
5631
5632 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5633 kMaxPayloadLength);
5634 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5635
5636 // The default bitrate limits for 360p should not be used.
5637 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005638 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5639 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005640 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5641 EXPECT_FALSE(WaitForFrame(1000));
5642 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005643 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005644 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5645 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5646 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5647 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5648 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5649 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5650
5651 video_stream_encoder_->Stop();
5652}
5653
5654TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5655 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5656 /*num_spatial_layers=*/1, /*screenshare=*/false);
5657
5658 // The default singlecast bitrate limits for 720p should not be used.
5659 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005660 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5661 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005662 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5663 EXPECT_FALSE(WaitForFrame(1000));
5664 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5665 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5666 VideoCodecType::kVideoCodecVP9);
5667 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5668 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5669 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5670 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5671 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5672 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5673
5674 video_stream_encoder_->Stop();
5675}
5676
5677TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005678 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5679 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5680 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5681 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5682 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5683 fake_encoder_.SetResolutionBitrateLimits(
5684 {kEncoderLimits180p, kEncoderLimits720p});
5685
5686 VideoEncoderConfig video_encoder_config;
5687 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5688 &video_encoder_config);
5689 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5690 vp9_settings.numberOfSpatialLayers = 3;
5691 // Since only one layer is active - automatic resize should be enabled.
5692 vp9_settings.automaticResizeOn = true;
5693 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005694 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005695 vp9_settings);
5696 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5697 video_encoder_config.content_type =
5698 VideoEncoderConfig::ContentType::kRealtimeVideo;
5699 // Simulcast layers are used to indicate which spatial layers are active.
5700 video_encoder_config.simulcast_layers.resize(3);
5701 video_encoder_config.simulcast_layers[0].active = true;
5702 video_encoder_config.simulcast_layers[1].active = false;
5703 video_encoder_config.simulcast_layers[2].active = false;
5704
5705 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5706 kMaxPayloadLength);
5707 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5708
5709 // Limits not applied on lowest stream, limits for 180p should not be used.
5710 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5711 EXPECT_FALSE(WaitForFrame(1000));
5712 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5713 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5714 VideoCodecType::kVideoCodecVP9);
5715 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5716 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5717 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5718 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5719 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5720 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5721 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5722 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5723
5724 video_stream_encoder_->Stop();
5725}
5726
5727TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005728 InitialFrameDropActivatesWhenResolutionIncreases) {
5729 const int kWidth = 640;
5730 const int kHeight = 360;
5731
5732 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5733 DataRate::BitsPerSec(kTargetBitrateBps),
5734 DataRate::BitsPerSec(kTargetBitrateBps),
5735 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5736 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5737 // Frame should not be dropped.
5738 WaitForEncodedFrame(1);
5739
5740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5741 DataRate::BitsPerSec(kLowTargetBitrateBps),
5742 DataRate::BitsPerSec(kLowTargetBitrateBps),
5743 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5744 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5745 // Frame should not be dropped, bitrate not too low for frame.
5746 WaitForEncodedFrame(2);
5747
5748 // Incoming resolution increases.
5749 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5750 // Expect to drop this frame, bitrate too low for frame.
5751 ExpectDroppedFrame();
5752
5753 // Expect the sink_wants to specify a scaled frame.
5754 EXPECT_TRUE_WAIT(
5755 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5756 video_stream_encoder_->Stop();
5757}
5758
5759TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5760 const int kWidth = 640;
5761 const int kHeight = 360;
5762 // So that quality scaling doesn't happen by itself.
5763 fake_encoder_.SetQp(kQpHigh);
5764
5765 AdaptingFrameForwarder source(&time_controller_);
5766 source.set_adaptation_enabled(true);
5767 video_stream_encoder_->SetSource(
5768 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5769
5770 int timestamp = 1;
5771
5772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5773 DataRate::BitsPerSec(kTargetBitrateBps),
5774 DataRate::BitsPerSec(kTargetBitrateBps),
5775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5776 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5777 WaitForEncodedFrame(timestamp);
5778 timestamp += 9000;
5779 // Long pause to disable all first BWE drop logic.
5780 AdvanceTime(TimeDelta::Millis(1000));
5781
5782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5783 DataRate::BitsPerSec(kLowTargetBitrateBps),
5784 DataRate::BitsPerSec(kLowTargetBitrateBps),
5785 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5786 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5787 // Not dropped frame, as initial frame drop is disabled by now.
5788 WaitForEncodedFrame(timestamp);
5789 timestamp += 9000;
5790 AdvanceTime(TimeDelta::Millis(100));
5791
5792 // Quality adaptation down.
5793 video_stream_encoder_->TriggerQualityLow();
5794
5795 // Adaptation has an effect.
5796 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5797 5000);
5798
5799 // Frame isn't dropped as initial frame dropper is disabled.
5800 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5801 WaitForEncodedFrame(timestamp);
5802 timestamp += 9000;
5803 AdvanceTime(TimeDelta::Millis(100));
5804
5805 // Quality adaptation up.
5806 video_stream_encoder_->TriggerQualityHigh();
5807
5808 // Adaptation has an effect.
5809 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5810 5000);
5811
5812 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5813 // Frame should not be dropped, as initial framedropper is off.
5814 WaitForEncodedFrame(timestamp);
5815
5816 video_stream_encoder_->Stop();
5817}
5818
Åsa Persson7f354f82021-02-04 15:52:15 +01005819TEST_F(VideoStreamEncoderTest,
5820 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5821 const int kMinStartBps360p = 222000;
5822 fake_encoder_.SetResolutionBitrateLimits(
5823 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5824 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5825 800000)});
5826
5827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5828 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5829 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5830 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5831 0, 0, 0);
5832 // Frame should not be dropped, bitrate not too low for frame.
5833 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5834 WaitForEncodedFrame(1);
5835
5836 // Incoming resolution increases, initial frame drop activates.
5837 // Frame should be dropped, link allocation too low for frame.
5838 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5839 ExpectDroppedFrame();
5840
5841 // Expect sink_wants to specify a scaled frame.
5842 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5843 5000);
5844 video_stream_encoder_->Stop();
5845}
5846
5847TEST_F(VideoStreamEncoderTest,
5848 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5849 const int kMinStartBps360p = 222000;
5850 fake_encoder_.SetResolutionBitrateLimits(
5851 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5852 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5853 800000)});
5854
5855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5856 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5857 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5858 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5859 0, 0, 0);
5860 // Frame should not be dropped, bitrate not too low for frame.
5861 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5862 WaitForEncodedFrame(1);
5863
5864 // Incoming resolution increases, initial frame drop activates.
5865 // Frame should be dropped, link allocation not too low for frame.
5866 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5867 WaitForEncodedFrame(2);
5868
5869 video_stream_encoder_->Stop();
5870}
5871
Åsa Perssone644a032019-11-08 15:56:00 +01005872TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5873 webrtc::test::ScopedFieldTrials field_trials(
5874 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5875
5876 // Reset encoder for field trials to take effect.
5877 VideoEncoderConfig config = video_encoder_config_.Copy();
5878 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005879 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005880 ConfigureEncoder(std::move(config));
5881 fake_encoder_.SetQp(kQpLow);
5882
5883 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005884 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005885 source.set_adaptation_enabled(true);
5886 video_stream_encoder_->SetSource(&source,
5887 DegradationPreference::MAINTAIN_FRAMERATE);
5888
5889 // Start at low bitrate.
5890 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5892 DataRate::BitsPerSec(kLowBitrateBps),
5893 DataRate::BitsPerSec(kLowBitrateBps),
5894 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005895
5896 // Expect first frame to be dropped and resolution to be limited.
5897 const int kWidth = 1280;
5898 const int kHeight = 720;
5899 const int64_t kFrameIntervalMs = 100;
5900 int64_t timestamp_ms = kFrameIntervalMs;
5901 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5902 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005903 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5904 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005905
5906 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5908 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005909
5910 // Insert frames and advance |min_duration_ms|.
5911 for (size_t i = 1; i <= 10; i++) {
5912 timestamp_ms += kFrameIntervalMs;
5913 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5914 WaitForEncodedFrame(timestamp_ms);
5915 }
5916 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5917 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5918
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005919 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005920
5921 // Insert frame should trigger high BW and release quality limitation.
5922 timestamp_ms += kFrameIntervalMs;
5923 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5924 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005925 // The ramp-up code involves the adaptation queue, give it time to execute.
5926 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005927 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005928 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005929
5930 // Frame should not be adapted.
5931 timestamp_ms += kFrameIntervalMs;
5932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5933 WaitForEncodedFrame(kWidth, kHeight);
5934 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5935
5936 video_stream_encoder_->Stop();
5937}
5938
mflodmancc3d4422017-08-03 08:27:51 -07005939TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005940 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005941 webrtc::test::ScopedFieldTrials field_trials(
5942 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005943 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005944 source.set_adaptation_enabled(true);
5945 video_stream_encoder_->SetSource(&source,
5946 DegradationPreference::MAINTAIN_FRAMERATE);
5947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5948 DataRate::BitsPerSec(kTargetBitrateBps),
5949 DataRate::BitsPerSec(kTargetBitrateBps),
5950 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5951 fake_encoder_.SetQp(kQpHigh + 1);
5952 const int kWidth = 1280;
5953 const int kHeight = 720;
5954 const int64_t kFrameIntervalMs = 100;
5955 int64_t timestamp_ms = kFrameIntervalMs;
5956 for (size_t i = 1; i <= 100; i++) {
5957 timestamp_ms += kFrameIntervalMs;
5958 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5959 WaitForEncodedFrame(timestamp_ms);
5960 }
5961 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5962 // for the first time.
5963 // TODO(eshr): We should avoid these waits by using threads with simulated
5964 // time.
5965 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5966 2000 * 2.5 * 2);
5967 timestamp_ms += kFrameIntervalMs;
5968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5969 WaitForEncodedFrame(timestamp_ms);
5970 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5971 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5972 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5973
5974 // Disable Quality scaling by turning off scaler on the encoder and
5975 // reconfiguring.
5976 fake_encoder_.SetQualityScaling(false);
5977 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5978 kMaxPayloadLength);
5979 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005980 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005981 // Since we turned off the quality scaler, the adaptations made by it are
5982 // removed.
5983 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5984 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5985
5986 video_stream_encoder_->Stop();
5987}
5988
5989TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005990 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5991 const int kTooSmallWidth = 10;
5992 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005994 DataRate::BitsPerSec(kTargetBitrateBps),
5995 DataRate::BitsPerSec(kTargetBitrateBps),
5996 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005997
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005998 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005999 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006000 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006001 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006002 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006003 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6004
6005 // Trigger adapt down, too small frame, expect no change.
6006 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006007 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006008 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006009 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006010 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6011 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6012
mflodmancc3d4422017-08-03 08:27:51 -07006013 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006014}
6015
mflodmancc3d4422017-08-03 08:27:51 -07006016TEST_F(VideoStreamEncoderTest,
6017 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006018 const int kTooSmallWidth = 10;
6019 const int kTooSmallHeight = 10;
6020 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006022 DataRate::BitsPerSec(kTargetBitrateBps),
6023 DataRate::BitsPerSec(kTargetBitrateBps),
6024 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006025
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006026 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006027 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006028 video_stream_encoder_->SetSource(&source,
6029 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006030 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006031 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6033
6034 // Trigger adapt down, expect limited framerate.
6035 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006036 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006037 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006038 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006039 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6040 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6041 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6042
6043 // Trigger adapt down, too small frame, expect no change.
6044 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006045 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006046 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006047 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006048 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6049 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6050 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6051
mflodmancc3d4422017-08-03 08:27:51 -07006052 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006053}
6054
mflodmancc3d4422017-08-03 08:27:51 -07006055TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006056 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006058 DataRate::BitsPerSec(kTargetBitrateBps),
6059 DataRate::BitsPerSec(kTargetBitrateBps),
6060 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006061 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006062 const int kFrameWidth = 1280;
6063 const int kFrameHeight = 720;
6064 video_source_.IncomingCapturedFrame(
6065 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006066 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006067 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006068}
6069
sprangb1ca0732017-02-01 08:38:12 -08006070// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006071TEST_F(VideoStreamEncoderTest,
6072 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006074 DataRate::BitsPerSec(kTargetBitrateBps),
6075 DataRate::BitsPerSec(kTargetBitrateBps),
6076 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006077
6078 const int kFrameWidth = 1280;
6079 const int kFrameHeight = 720;
6080 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006081 // requested by
6082 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006083 video_source_.set_adaptation_enabled(true);
6084
6085 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006086 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006087 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006088
6089 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006090 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006091 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006092 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006093 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006094
asaperssonfab67072017-04-04 05:51:49 -07006095 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006096 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006097 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006098 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006099 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006100
mflodmancc3d4422017-08-03 08:27:51 -07006101 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006102}
sprangfe627f32017-03-29 08:24:59 -07006103
mflodmancc3d4422017-08-03 08:27:51 -07006104TEST_F(VideoStreamEncoderTest,
6105 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006106 const int kFrameWidth = 1280;
6107 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006108
Henrik Boström381d1092020-05-12 18:49:07 +02006109 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006110 DataRate::BitsPerSec(kTargetBitrateBps),
6111 DataRate::BitsPerSec(kTargetBitrateBps),
6112 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006113 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006114 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006115 video_source_.set_adaptation_enabled(true);
6116
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006117 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006118
6119 video_source_.IncomingCapturedFrame(
6120 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006121 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006122
6123 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006124 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006125
6126 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006127 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006128 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006129 video_source_.IncomingCapturedFrame(
6130 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006131 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006132 }
6133
6134 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006135 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006136 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006137 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006138 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006139 video_source_.IncomingCapturedFrame(
6140 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006141 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006142 ++num_frames_dropped;
6143 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006144 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006145 }
6146 }
6147
sprang4847ae62017-06-27 07:06:52 -07006148 // Add some slack to account for frames dropped by the frame dropper.
6149 const int kErrorMargin = 1;
6150 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006151 kErrorMargin);
6152
6153 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006154 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006155 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006156 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006157 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006158 video_source_.IncomingCapturedFrame(
6159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006160 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006161 ++num_frames_dropped;
6162 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006163 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006164 }
6165 }
sprang4847ae62017-06-27 07:06:52 -07006166 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006167 kErrorMargin);
6168
6169 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006170 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006171 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006172 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006173 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006174 video_source_.IncomingCapturedFrame(
6175 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006176 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006177 ++num_frames_dropped;
6178 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006179 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006180 }
6181 }
sprang4847ae62017-06-27 07:06:52 -07006182 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006183 kErrorMargin);
6184
6185 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006186 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006187 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006188 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006189 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006190 video_source_.IncomingCapturedFrame(
6191 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006192 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006193 ++num_frames_dropped;
6194 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006195 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006196 }
6197 }
6198 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6199
mflodmancc3d4422017-08-03 08:27:51 -07006200 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006201}
6202
mflodmancc3d4422017-08-03 08:27:51 -07006203TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006204 const int kFramerateFps = 5;
6205 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006206 const int kFrameWidth = 1280;
6207 const int kFrameHeight = 720;
6208
sprang4847ae62017-06-27 07:06:52 -07006209 // Reconfigure encoder with two temporal layers and screensharing, which will
6210 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006211 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006212
Henrik Boström381d1092020-05-12 18:49:07 +02006213 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006214 DataRate::BitsPerSec(kTargetBitrateBps),
6215 DataRate::BitsPerSec(kTargetBitrateBps),
6216 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006217 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006218 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006219 video_source_.set_adaptation_enabled(true);
6220
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006221 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006222
6223 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006224 rtc::VideoSinkWants last_wants;
6225 do {
6226 last_wants = video_source_.sink_wants();
6227
sprangc5d62e22017-04-02 23:53:04 -07006228 // Insert frames to get a new fps estimate...
6229 for (int j = 0; j < kFramerateFps; ++j) {
6230 video_source_.IncomingCapturedFrame(
6231 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006232 if (video_source_.last_sent_width()) {
6233 sink_.WaitForEncodedFrame(timestamp_ms);
6234 }
sprangc5d62e22017-04-02 23:53:04 -07006235 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006236 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006237 }
6238 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006239 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006240 } while (video_source_.sink_wants().max_framerate_fps <
6241 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006242
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006243 EXPECT_THAT(video_source_.sink_wants(),
6244 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006245
mflodmancc3d4422017-08-03 08:27:51 -07006246 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006247}
asaperssonf7e294d2017-06-13 23:25:22 -07006248
mflodmancc3d4422017-08-03 08:27:51 -07006249TEST_F(VideoStreamEncoderTest,
6250 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006251 const int kWidth = 1280;
6252 const int kHeight = 720;
6253 const int64_t kFrameIntervalMs = 150;
6254 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006255 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006256 DataRate::BitsPerSec(kTargetBitrateBps),
6257 DataRate::BitsPerSec(kTargetBitrateBps),
6258 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006259
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006260 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006261 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006262 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006263 video_stream_encoder_->SetSource(&source,
6264 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006265 timestamp_ms += kFrameIntervalMs;
6266 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006267 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006268 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006269 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6271 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6272
6273 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006274 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006275 timestamp_ms += kFrameIntervalMs;
6276 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006277 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006278 EXPECT_THAT(source.sink_wants(),
6279 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006280 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6281 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6282 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6283
6284 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006285 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006286 timestamp_ms += kFrameIntervalMs;
6287 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006288 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006289 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006290 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6292 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6293
6294 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006295 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006296 timestamp_ms += kFrameIntervalMs;
6297 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006298 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006299 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006300 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6301 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6302 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6303
6304 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006305 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006306 timestamp_ms += kFrameIntervalMs;
6307 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006308 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006309 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006310 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6312 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6313
6314 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006315 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006316 timestamp_ms += kFrameIntervalMs;
6317 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006318 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006319 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006320 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6321 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6322 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6323
6324 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006325 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006326 timestamp_ms += kFrameIntervalMs;
6327 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006328 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006329 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006330 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6331 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6332 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6333
6334 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006335 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006336 timestamp_ms += kFrameIntervalMs;
6337 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006338 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006339 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006340 rtc::VideoSinkWants last_wants = source.sink_wants();
6341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6342 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6343 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6344
6345 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006346 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006347 timestamp_ms += kFrameIntervalMs;
6348 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006349 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006350 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006351 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6352 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6353 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6354
Evan Shrubsole64469032020-06-11 10:45:29 +02006355 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006356 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006357 timestamp_ms += kFrameIntervalMs;
6358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006359 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006360 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006361 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6362 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6363 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6364
6365 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006366 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006367 timestamp_ms += kFrameIntervalMs;
6368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006369 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006370 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006371 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6372 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6373 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6374
6375 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006376 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006377 timestamp_ms += kFrameIntervalMs;
6378 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006379 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006380 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006381 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6382 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6383 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6384
6385 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006386 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006387 timestamp_ms += kFrameIntervalMs;
6388 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006389 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006390 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006391 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6392 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6393 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6394
6395 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006396 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006397 timestamp_ms += kFrameIntervalMs;
6398 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006399 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006400 EXPECT_THAT(source.sink_wants(), FpsMax());
6401 EXPECT_EQ(source.sink_wants().max_pixel_count,
6402 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006403 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6404 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6405 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6406
6407 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006408 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006409 timestamp_ms += kFrameIntervalMs;
6410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006411 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006412 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6414 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6415 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6416
Åsa Persson30ab0152019-08-27 12:22:33 +02006417 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006418 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006419 timestamp_ms += kFrameIntervalMs;
6420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006421 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006422 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006423 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006424 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6425 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6426 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6427
6428 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006429 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006430 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006431 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6432
mflodmancc3d4422017-08-03 08:27:51 -07006433 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006434}
6435
mflodmancc3d4422017-08-03 08:27:51 -07006436TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006437 const int kWidth = 1280;
6438 const int kHeight = 720;
6439 const int64_t kFrameIntervalMs = 150;
6440 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006442 DataRate::BitsPerSec(kTargetBitrateBps),
6443 DataRate::BitsPerSec(kTargetBitrateBps),
6444 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006445
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006446 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006447 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006448 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006449 video_stream_encoder_->SetSource(&source,
6450 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006451 timestamp_ms += kFrameIntervalMs;
6452 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006453 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006454 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6456 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6457 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6458 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6461
6462 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006463 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006464 timestamp_ms += kFrameIntervalMs;
6465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006466 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006467 EXPECT_THAT(source.sink_wants(),
6468 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6470 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6471 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6472 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6473 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6474 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6475
6476 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006477 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006478 timestamp_ms += kFrameIntervalMs;
6479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006480 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006481 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006482 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6483 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6484 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6485 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6486 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6487 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6488
6489 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006490 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006491 timestamp_ms += kFrameIntervalMs;
6492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006493 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006494 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6497 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6498 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6499 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6500 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6501
Evan Shrubsole64469032020-06-11 10:45:29 +02006502 // Trigger cpu adapt up, expect no change since QP is most limited.
6503 {
6504 // Store current sink wants since we expect no change and if there is no
6505 // change then last_wants() is not updated.
6506 auto previous_sink_wants = source.sink_wants();
6507 video_stream_encoder_->TriggerCpuUnderuse();
6508 timestamp_ms += kFrameIntervalMs;
6509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6510 WaitForEncodedFrame(timestamp_ms);
6511 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6512 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6513 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6514 }
6515
6516 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6517 video_stream_encoder_->TriggerQualityHigh();
6518 timestamp_ms += kFrameIntervalMs;
6519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6520 WaitForEncodedFrame(timestamp_ms);
6521 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6525 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6526 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6527 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6528
6529 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6530 // expect increased resolution (960x540@30fps).
6531 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006532 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006533 timestamp_ms += kFrameIntervalMs;
6534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006536 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6539 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6540 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6541 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006542 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006543
Evan Shrubsole64469032020-06-11 10:45:29 +02006544 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6545 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006546 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006547 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006548 timestamp_ms += kFrameIntervalMs;
6549 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006550 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006551 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6555 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6557 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006558 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006559
6560 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006562 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006563 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006564 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006565
mflodmancc3d4422017-08-03 08:27:51 -07006566 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006567}
6568
mflodmancc3d4422017-08-03 08:27:51 -07006569TEST_F(VideoStreamEncoderTest,
6570 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006571 const int kWidth = 640;
6572 const int kHeight = 360;
6573 const int kFpsLimit = 15;
6574 const int64_t kFrameIntervalMs = 150;
6575 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006577 DataRate::BitsPerSec(kTargetBitrateBps),
6578 DataRate::BitsPerSec(kTargetBitrateBps),
6579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006580
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006581 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006582 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006583 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006584 video_stream_encoder_->SetSource(&source,
6585 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006586 timestamp_ms += kFrameIntervalMs;
6587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006588 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006589 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006590 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6591 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6592 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6593 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6595 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6596
6597 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006598 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006599 timestamp_ms += kFrameIntervalMs;
6600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006601 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006602 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006603 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6604 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6605 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6606 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6607 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6608 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
6610 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006618 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6619 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6621 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6622
Evan Shrubsole64469032020-06-11 10:45:29 +02006623 // Trigger cpu adapt up, expect no change because quality is most limited.
6624 {
6625 auto previous_sink_wants = source.sink_wants();
6626 // Store current sink wants since we expect no change ind if there is no
6627 // change then last__wants() is not updated.
6628 video_stream_encoder_->TriggerCpuUnderuse();
6629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6631 WaitForEncodedFrame(timestamp_ms);
6632 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6633 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6634 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6635 }
6636
6637 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6638 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006639 timestamp_ms += kFrameIntervalMs;
6640 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006641 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006642 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6644 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6645 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006646 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6647 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6648 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006649
Evan Shrubsole64469032020-06-11 10:45:29 +02006650 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006651 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006652 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006653 timestamp_ms += kFrameIntervalMs;
6654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006656 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6660 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6661 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006662 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006663
6664 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006665 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006667 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006668 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006669
mflodmancc3d4422017-08-03 08:27:51 -07006670 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006671}
6672
mflodmancc3d4422017-08-03 08:27:51 -07006673TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006674 const int kFrameWidth = 1920;
6675 const int kFrameHeight = 1080;
6676 // 3/4 of 1920.
6677 const int kAdaptedFrameWidth = 1440;
6678 // 3/4 of 1080 rounded down to multiple of 4.
6679 const int kAdaptedFrameHeight = 808;
6680 const int kFramerate = 24;
6681
Henrik Boström381d1092020-05-12 18:49:07 +02006682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006683 DataRate::BitsPerSec(kTargetBitrateBps),
6684 DataRate::BitsPerSec(kTargetBitrateBps),
6685 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006686 // Trigger reconfigure encoder (without resetting the entire instance).
6687 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006688 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6689 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006690 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006691 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006692 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006693 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006694 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006695 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006696
6697 video_source_.set_adaptation_enabled(true);
6698
6699 video_source_.IncomingCapturedFrame(
6700 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006701 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006702
6703 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006704 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006705 video_source_.IncomingCapturedFrame(
6706 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006707 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006708
mflodmancc3d4422017-08-03 08:27:51 -07006709 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006710}
6711
mflodmancc3d4422017-08-03 08:27:51 -07006712TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006713 const int kFrameWidth = 1280;
6714 const int kFrameHeight = 720;
6715 const int kLowFps = 2;
6716 const int kHighFps = 30;
6717
Henrik Boström381d1092020-05-12 18:49:07 +02006718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006719 DataRate::BitsPerSec(kTargetBitrateBps),
6720 DataRate::BitsPerSec(kTargetBitrateBps),
6721 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006722
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006723 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006724 max_framerate_ = kLowFps;
6725
6726 // Insert 2 seconds of 2fps video.
6727 for (int i = 0; i < kLowFps * 2; ++i) {
6728 video_source_.IncomingCapturedFrame(
6729 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6730 WaitForEncodedFrame(timestamp_ms);
6731 timestamp_ms += 1000 / kLowFps;
6732 }
6733
6734 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006736 DataRate::BitsPerSec(kTargetBitrateBps),
6737 DataRate::BitsPerSec(kTargetBitrateBps),
6738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006739 video_source_.IncomingCapturedFrame(
6740 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6741 WaitForEncodedFrame(timestamp_ms);
6742 timestamp_ms += 1000 / kLowFps;
6743
6744 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6745
6746 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006747 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006748 const int kFrameIntervalMs = 1000 / kHighFps;
6749 max_framerate_ = kHighFps;
6750 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6751 video_source_.IncomingCapturedFrame(
6752 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6753 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6754 // be dropped if the encoder hans't been updated with the new higher target
6755 // framerate yet, causing it to overshoot the target bitrate and then
6756 // suffering the wrath of the media optimizer.
6757 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6758 timestamp_ms += kFrameIntervalMs;
6759 }
6760
6761 // Don expect correct measurement just yet, but it should be higher than
6762 // before.
6763 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6764
mflodmancc3d4422017-08-03 08:27:51 -07006765 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006766}
6767
mflodmancc3d4422017-08-03 08:27:51 -07006768TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006769 const int kFrameWidth = 1280;
6770 const int kFrameHeight = 720;
6771 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006772 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006773 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006774 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006775
Henrik Boström381d1092020-05-12 18:49:07 +02006776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006777 DataRate::BitsPerSec(kTargetBitrateBps),
6778 DataRate::BitsPerSec(kTargetBitrateBps),
6779 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006780 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006781
6782 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006783 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006784 video_source_.IncomingCapturedFrame(
6785 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6786 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006787 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006788
6789 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6791 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6792 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006793
6794 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006795 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006796 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006797
Per Kjellanderdcef6412020-10-07 15:09:05 +02006798 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006799 video_source_.IncomingCapturedFrame(
6800 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6801 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006802 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006803
mflodmancc3d4422017-08-03 08:27:51 -07006804 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006805}
ilnik6b826ef2017-06-16 06:53:48 -07006806
Niels Möller4db138e2018-04-19 09:04:13 +02006807TEST_F(VideoStreamEncoderTest,
6808 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6809 const int kFrameWidth = 1280;
6810 const int kFrameHeight = 720;
6811 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006813 DataRate::BitsPerSec(kTargetBitrateBps),
6814 DataRate::BitsPerSec(kTargetBitrateBps),
6815 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006816 video_source_.IncomingCapturedFrame(
6817 CreateFrame(1, kFrameWidth, kFrameHeight));
6818 WaitForEncodedFrame(1);
6819 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6820 .low_encode_usage_threshold_percent,
6821 default_options.low_encode_usage_threshold_percent);
6822 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6823 .high_encode_usage_threshold_percent,
6824 default_options.high_encode_usage_threshold_percent);
6825 video_stream_encoder_->Stop();
6826}
6827
6828TEST_F(VideoStreamEncoderTest,
6829 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6830 const int kFrameWidth = 1280;
6831 const int kFrameHeight = 720;
6832 CpuOveruseOptions hardware_options;
6833 hardware_options.low_encode_usage_threshold_percent = 150;
6834 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006835 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006836
Henrik Boström381d1092020-05-12 18:49:07 +02006837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006838 DataRate::BitsPerSec(kTargetBitrateBps),
6839 DataRate::BitsPerSec(kTargetBitrateBps),
6840 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006841 video_source_.IncomingCapturedFrame(
6842 CreateFrame(1, kFrameWidth, kFrameHeight));
6843 WaitForEncodedFrame(1);
6844 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6845 .low_encode_usage_threshold_percent,
6846 hardware_options.low_encode_usage_threshold_percent);
6847 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6848 .high_encode_usage_threshold_percent,
6849 hardware_options.high_encode_usage_threshold_percent);
6850 video_stream_encoder_->Stop();
6851}
6852
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006853TEST_F(VideoStreamEncoderTest,
6854 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6855 const int kFrameWidth = 1280;
6856 const int kFrameHeight = 720;
6857
6858 const CpuOveruseOptions default_options;
6859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6860 DataRate::BitsPerSec(kTargetBitrateBps),
6861 DataRate::BitsPerSec(kTargetBitrateBps),
6862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6863 video_source_.IncomingCapturedFrame(
6864 CreateFrame(1, kFrameWidth, kFrameHeight));
6865 WaitForEncodedFrame(1);
6866 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6867 .low_encode_usage_threshold_percent,
6868 default_options.low_encode_usage_threshold_percent);
6869 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6870 .high_encode_usage_threshold_percent,
6871 default_options.high_encode_usage_threshold_percent);
6872
6873 CpuOveruseOptions hardware_options;
6874 hardware_options.low_encode_usage_threshold_percent = 150;
6875 hardware_options.high_encode_usage_threshold_percent = 200;
6876 fake_encoder_.SetIsHardwareAccelerated(true);
6877
6878 video_source_.IncomingCapturedFrame(
6879 CreateFrame(2, kFrameWidth, kFrameHeight));
6880 WaitForEncodedFrame(2);
6881
6882 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6883 .low_encode_usage_threshold_percent,
6884 hardware_options.low_encode_usage_threshold_percent);
6885 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6886 .high_encode_usage_threshold_percent,
6887 hardware_options.high_encode_usage_threshold_percent);
6888
6889 video_stream_encoder_->Stop();
6890}
6891
Niels Möller6bb5ab92019-01-11 11:11:10 +01006892TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6893 const int kFrameWidth = 320;
6894 const int kFrameHeight = 240;
6895 const int kFps = 30;
6896 const int kTargetBitrateBps = 120000;
6897 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6898
Henrik Boström381d1092020-05-12 18:49:07 +02006899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006900 DataRate::BitsPerSec(kTargetBitrateBps),
6901 DataRate::BitsPerSec(kTargetBitrateBps),
6902 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006903
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006904 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006905 max_framerate_ = kFps;
6906
6907 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6908 fake_encoder_.SimulateOvershoot(1.0);
6909 int num_dropped = 0;
6910 for (int i = 0; i < kNumFramesInRun; ++i) {
6911 video_source_.IncomingCapturedFrame(
6912 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6913 // Wait up to two frame durations for a frame to arrive.
6914 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6915 ++num_dropped;
6916 }
6917 timestamp_ms += 1000 / kFps;
6918 }
6919
Erik Språnga8d48ab2019-02-08 14:17:40 +01006920 // Framerate should be measured to be near the expected target rate.
6921 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6922
6923 // Frame drops should be within 5% of expected 0%.
6924 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006925
6926 // Make encoder produce frames at double the expected bitrate during 3 seconds
6927 // of video, verify number of drops. Rate needs to be slightly changed in
6928 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006929 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006930 const RateControlSettings trials =
6931 RateControlSettings::ParseFromFieldTrials();
6932 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006933 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006934 // frame dropping since the adjuter will try to just lower the target
6935 // bitrate rather than drop frames. If network headroom can be used, it
6936 // doesn't push back as hard so we don't need quite as much overshoot.
6937 // These numbers are unfortunately a bit magical but there's not trivial
6938 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01006939 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01006940 }
6941 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006943 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6944 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6945 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006946 num_dropped = 0;
6947 for (int i = 0; i < kNumFramesInRun; ++i) {
6948 video_source_.IncomingCapturedFrame(
6949 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6950 // Wait up to two frame durations for a frame to arrive.
6951 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6952 ++num_dropped;
6953 }
6954 timestamp_ms += 1000 / kFps;
6955 }
6956
Henrik Boström381d1092020-05-12 18:49:07 +02006957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006958 DataRate::BitsPerSec(kTargetBitrateBps),
6959 DataRate::BitsPerSec(kTargetBitrateBps),
6960 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006961
6962 // Target framerate should be still be near the expected target, despite
6963 // the frame drops.
6964 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6965
6966 // Frame drops should be within 5% of expected 50%.
6967 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006968
6969 video_stream_encoder_->Stop();
6970}
6971
6972TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6973 const int kFrameWidth = 320;
6974 const int kFrameHeight = 240;
6975 const int kActualInputFps = 24;
6976 const int kTargetBitrateBps = 120000;
6977
6978 ASSERT_GT(max_framerate_, kActualInputFps);
6979
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006980 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006981 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006983 DataRate::BitsPerSec(kTargetBitrateBps),
6984 DataRate::BitsPerSec(kTargetBitrateBps),
6985 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006986
6987 // Insert 3 seconds of video, with an input fps lower than configured max.
6988 for (int i = 0; i < kActualInputFps * 3; ++i) {
6989 video_source_.IncomingCapturedFrame(
6990 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6991 // Wait up to two frame durations for a frame to arrive.
6992 WaitForEncodedFrame(timestamp_ms);
6993 timestamp_ms += 1000 / kActualInputFps;
6994 }
6995
6996 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6997
6998 video_stream_encoder_->Stop();
6999}
7000
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007001TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007002 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007004 DataRate::BitsPerSec(kTargetBitrateBps),
7005 DataRate::BitsPerSec(kTargetBitrateBps),
7006 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007007
7008 fake_encoder_.BlockNextEncode();
7009 video_source_.IncomingCapturedFrame(
7010 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7011 WaitForEncodedFrame(1);
7012 // On the very first frame full update should be forced.
7013 rect = fake_encoder_.GetLastUpdateRect();
7014 EXPECT_EQ(rect.offset_x, 0);
7015 EXPECT_EQ(rect.offset_y, 0);
7016 EXPECT_EQ(rect.height, codec_height_);
7017 EXPECT_EQ(rect.width, codec_width_);
7018 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7019 // call to ContinueEncode.
7020 video_source_.IncomingCapturedFrame(
7021 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7022 ExpectDroppedFrame();
7023 video_source_.IncomingCapturedFrame(
7024 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7025 ExpectDroppedFrame();
7026 fake_encoder_.ContinueEncode();
7027 WaitForEncodedFrame(3);
7028 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7029 rect = fake_encoder_.GetLastUpdateRect();
7030 EXPECT_EQ(rect.offset_x, 1);
7031 EXPECT_EQ(rect.offset_y, 0);
7032 EXPECT_EQ(rect.width, 10);
7033 EXPECT_EQ(rect.height, 1);
7034
7035 video_source_.IncomingCapturedFrame(
7036 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7037 WaitForEncodedFrame(4);
7038 // Previous frame was encoded, so no accumulation should happen.
7039 rect = fake_encoder_.GetLastUpdateRect();
7040 EXPECT_EQ(rect.offset_x, 0);
7041 EXPECT_EQ(rect.offset_y, 0);
7042 EXPECT_EQ(rect.width, 1);
7043 EXPECT_EQ(rect.height, 1);
7044
7045 video_stream_encoder_->Stop();
7046}
7047
Erik Språngd7329ca2019-02-21 21:19:53 +01007048TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007050 DataRate::BitsPerSec(kTargetBitrateBps),
7051 DataRate::BitsPerSec(kTargetBitrateBps),
7052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007053
7054 // First frame is always keyframe.
7055 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7056 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007057 EXPECT_THAT(
7058 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007059 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007060
7061 // Insert delta frame.
7062 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7063 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007064 EXPECT_THAT(
7065 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007066 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007067
7068 // Request next frame be a key-frame.
7069 video_stream_encoder_->SendKeyFrame();
7070 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7071 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007072 EXPECT_THAT(
7073 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007074 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007075
7076 video_stream_encoder_->Stop();
7077}
7078
7079TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7080 // Setup simulcast with three streams.
7081 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007082 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007083 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7084 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7085 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007086 // Wait for all three layers before triggering event.
7087 sink_.SetNumExpectedLayers(3);
7088
7089 // First frame is always keyframe.
7090 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7091 WaitForEncodedFrame(1);
7092 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007093 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7094 VideoFrameType::kVideoFrameKey,
7095 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007096
7097 // Insert delta frame.
7098 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7099 WaitForEncodedFrame(2);
7100 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007101 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7102 VideoFrameType::kVideoFrameDelta,
7103 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007104
7105 // Request next frame be a key-frame.
7106 // Only first stream is configured to produce key-frame.
7107 video_stream_encoder_->SendKeyFrame();
7108 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7109 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007110
7111 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7112 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007113 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007114 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007115 VideoFrameType::kVideoFrameKey,
7116 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007117
7118 video_stream_encoder_->Stop();
7119}
7120
7121TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7122 // Configure internal source factory and setup test again.
7123 encoder_factory_.SetHasInternalSource(true);
7124 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007126 DataRate::BitsPerSec(kTargetBitrateBps),
7127 DataRate::BitsPerSec(kTargetBitrateBps),
7128 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007129
7130 // Call encoder directly, simulating internal source where encoded frame
7131 // callback in VideoStreamEncoder is called despite no OnFrame().
7132 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7133 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007134 EXPECT_THAT(
7135 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007136 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007137
Niels Möller8f7ce222019-03-21 15:43:58 +01007138 const std::vector<VideoFrameType> kDeltaFrame = {
7139 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007140 // Need to set timestamp manually since manually for injected frame.
7141 VideoFrame frame = CreateFrame(101, nullptr);
7142 frame.set_timestamp(101);
7143 fake_encoder_.InjectFrame(frame, false);
7144 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007145 EXPECT_THAT(
7146 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007147 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007148
7149 // Request key-frame. The forces a dummy frame down into the encoder.
7150 fake_encoder_.ExpectNullFrame();
7151 video_stream_encoder_->SendKeyFrame();
7152 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007153 EXPECT_THAT(
7154 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007155 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007156
7157 video_stream_encoder_->Stop();
7158}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007159
7160TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7161 // Configure internal source factory and setup test again.
7162 encoder_factory_.SetHasInternalSource(true);
7163 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007164 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007165 DataRate::BitsPerSec(kTargetBitrateBps),
7166 DataRate::BitsPerSec(kTargetBitrateBps),
7167 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007168
7169 int64_t timestamp = 1;
7170 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007171 image.capture_time_ms_ = ++timestamp;
7172 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7173 const int64_t kEncodeFinishDelayMs = 10;
7174 image.timing_.encode_start_ms = timestamp;
7175 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007176 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007177 // Wait for frame without incrementing clock.
7178 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7179 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7180 // capture timestamp should be kEncodeFinishDelayMs in the past.
7181 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007182 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007183
7184 video_stream_encoder_->Stop();
7185}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007186
7187TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007188 // SPS contains VUI with restrictions on the maximum number of reordered
7189 // pictures, there is no need to rewrite the bitstream to enable faster
7190 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007191 ResetEncoder("H264", 1, 1, 1, false);
7192
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7194 DataRate::BitsPerSec(kTargetBitrateBps),
7195 DataRate::BitsPerSec(kTargetBitrateBps),
7196 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7197 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007198
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007199 fake_encoder_.SetEncodedImageData(
7200 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007201
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007202 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7203 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007204
7205 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7206 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007207
7208 video_stream_encoder_->Stop();
7209}
7210
7211TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007212 // SPS does not contain VUI, the bitstream is will be rewritten with added
7213 // VUI with restrictions on the maximum number of reordered pictures to
7214 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007215 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7216 0x00, 0x00, 0x03, 0x03, 0xF4,
7217 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007218 ResetEncoder("H264", 1, 1, 1, false);
7219
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7221 DataRate::BitsPerSec(kTargetBitrateBps),
7222 DataRate::BitsPerSec(kTargetBitrateBps),
7223 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7224 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007225
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007226 fake_encoder_.SetEncodedImageData(
7227 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007228
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007229 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7230 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007231
7232 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7233 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007234
7235 video_stream_encoder_->Stop();
7236}
7237
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007238TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7239 const int kFrameWidth = 1280;
7240 const int kFrameHeight = 720;
7241 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7242
Henrik Boström381d1092020-05-12 18:49:07 +02007243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007244 DataRate::BitsPerSec(kTargetBitrateBps),
7245 DataRate::BitsPerSec(kTargetBitrateBps),
7246 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007247 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7248
7249 // Insert a first video frame. It should be dropped because of downscale in
7250 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007251 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007252 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7253 frame.set_rotation(kVideoRotation_270);
7254 video_source_.IncomingCapturedFrame(frame);
7255
7256 ExpectDroppedFrame();
7257
7258 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007259 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007260 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7261 frame.set_rotation(kVideoRotation_90);
7262 video_source_.IncomingCapturedFrame(frame);
7263
7264 WaitForEncodedFrame(timestamp_ms);
7265 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7266
7267 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007268 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007269 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7270 frame.set_rotation(kVideoRotation_180);
7271 video_source_.IncomingCapturedFrame(frame);
7272
7273 WaitForEncodedFrame(timestamp_ms);
7274 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7275
7276 video_stream_encoder_->Stop();
7277}
7278
Erik Språng5056af02019-09-02 15:53:11 +02007279TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7280 const int kFrameWidth = 320;
7281 const int kFrameHeight = 180;
7282
7283 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007285 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7286 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7287 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007288 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007289 /*rtt_ms=*/0,
7290 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007291
7292 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007293 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007294 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7295 frame.set_rotation(kVideoRotation_270);
7296 video_source_.IncomingCapturedFrame(frame);
7297 WaitForEncodedFrame(timestamp_ms);
7298
7299 // Set a target rate below the minimum allowed by the codec settings.
7300 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007301 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7302 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007304 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007305 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007306 /*link_allocation=*/target_rate,
7307 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007308 /*rtt_ms=*/0,
7309 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007310 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7311
7312 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7313 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7314 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007315 DataRate allocation_sum =
7316 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007317 EXPECT_EQ(min_rate, allocation_sum);
7318 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7319
7320 video_stream_encoder_->Stop();
7321}
7322
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007323TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007325 DataRate::BitsPerSec(kTargetBitrateBps),
7326 DataRate::BitsPerSec(kTargetBitrateBps),
7327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007328 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007329 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007330 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7331 WaitForEncodedFrame(1);
7332
7333 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7334 ASSERT_TRUE(prev_rate_settings.has_value());
7335 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7336 kDefaultFramerate);
7337
7338 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7339 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7340 timestamp_ms += 1000 / kDefaultFramerate;
7341 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7342 WaitForEncodedFrame(timestamp_ms);
7343 }
7344 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7345 kDefaultFramerate);
7346 // Capture larger frame to trigger a reconfigure.
7347 codec_height_ *= 2;
7348 codec_width_ *= 2;
7349 timestamp_ms += 1000 / kDefaultFramerate;
7350 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7351 WaitForEncodedFrame(timestamp_ms);
7352
7353 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7354 auto current_rate_settings =
7355 fake_encoder_.GetAndResetLastRateControlSettings();
7356 // Ensure we have actually reconfigured twice
7357 // The rate settings should have been set again even though
7358 // they haven't changed.
7359 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007360 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007361
7362 video_stream_encoder_->Stop();
7363}
7364
philipeld9cc8c02019-09-16 14:53:40 +02007365struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007366 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7367 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7368 MOCK_METHOD(void,
7369 RequestEncoderSwitch,
7370 (const webrtc::SdpVideoFormat& format),
7371 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007372};
7373
philipel9b058032020-02-10 11:30:00 +01007374TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7375 constexpr int kDontCare = 100;
7376 StrictMock<MockEncoderSelector> encoder_selector;
7377 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7378 &fake_encoder_, &encoder_selector);
7379 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7380
7381 // Reset encoder for new configuration to take effect.
7382 ConfigureEncoder(video_encoder_config_.Copy());
7383
7384 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7385
7386 video_source_.IncomingCapturedFrame(
7387 CreateFrame(kDontCare, kDontCare, kDontCare));
7388 video_stream_encoder_->Stop();
7389
7390 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7391 // to it's factory, so in order for the encoder instance in the
7392 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7393 // reset the |video_stream_encoder_| here.
7394 video_stream_encoder_.reset();
7395}
7396
7397TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7398 constexpr int kDontCare = 100;
7399
7400 NiceMock<MockEncoderSelector> encoder_selector;
7401 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7402 video_send_config_.encoder_settings.encoder_switch_request_callback =
7403 &switch_callback;
7404 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7405 &fake_encoder_, &encoder_selector);
7406 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7407
7408 // Reset encoder for new configuration to take effect.
7409 ConfigureEncoder(video_encoder_config_.Copy());
7410
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007411 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007412 .WillByDefault(Return(SdpVideoFormat("AV1")));
7413 EXPECT_CALL(switch_callback,
7414 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7415 Field(&SdpVideoFormat::name, "AV1"))));
7416
Henrik Boström381d1092020-05-12 18:49:07 +02007417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007418 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7419 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7420 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007421 /*fraction_lost=*/0,
7422 /*rtt_ms=*/0,
7423 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007424 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007425
7426 video_stream_encoder_->Stop();
7427}
7428
7429TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7430 constexpr int kSufficientBitrateToNotDrop = 1000;
7431 constexpr int kDontCare = 100;
7432
7433 NiceMock<MockVideoEncoder> video_encoder;
7434 NiceMock<MockEncoderSelector> encoder_selector;
7435 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7436 video_send_config_.encoder_settings.encoder_switch_request_callback =
7437 &switch_callback;
7438 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7439 &video_encoder, &encoder_selector);
7440 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7441
7442 // Reset encoder for new configuration to take effect.
7443 ConfigureEncoder(video_encoder_config_.Copy());
7444
7445 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7446 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7447 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007448 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007449 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7450 /*stable_target_bitrate=*/
7451 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7452 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007453 /*fraction_lost=*/0,
7454 /*rtt_ms=*/0,
7455 /*cwnd_reduce_ratio=*/0);
7456
7457 ON_CALL(video_encoder, Encode(_, _))
7458 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7459 ON_CALL(encoder_selector, OnEncoderBroken())
7460 .WillByDefault(Return(SdpVideoFormat("AV2")));
7461
7462 rtc::Event encode_attempted;
7463 EXPECT_CALL(switch_callback,
7464 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7465 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7466 EXPECT_EQ(format.name, "AV2");
7467 encode_attempted.Set();
7468 });
7469
7470 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7471 encode_attempted.Wait(3000);
7472
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007473 AdvanceTime(TimeDelta::Millis(0));
7474
philipel9b058032020-02-10 11:30:00 +01007475 video_stream_encoder_->Stop();
7476
7477 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7478 // to it's factory, so in order for the encoder instance in the
7479 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7480 // reset the |video_stream_encoder_| here.
7481 video_stream_encoder_.reset();
7482}
7483
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007484TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007485 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007486 const int kFrameWidth = 320;
7487 const int kFrameHeight = 180;
7488
7489 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007490 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007492 /*target_bitrate=*/rate,
7493 /*stable_target_bitrate=*/rate,
7494 /*link_allocation=*/rate,
7495 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007496 /*rtt_ms=*/0,
7497 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007498
7499 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007500 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007501 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7502 frame.set_rotation(kVideoRotation_270);
7503 video_source_.IncomingCapturedFrame(frame);
7504 WaitForEncodedFrame(timestamp_ms);
7505 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7506
7507 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007508 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007510 /*target_bitrate=*/new_stable_rate,
7511 /*stable_target_bitrate=*/new_stable_rate,
7512 /*link_allocation=*/rate,
7513 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007514 /*rtt_ms=*/0,
7515 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007516 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7517 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7518 video_stream_encoder_->Stop();
7519}
7520
7521TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007522 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007523 const int kFrameWidth = 320;
7524 const int kFrameHeight = 180;
7525
7526 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007527 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007528 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007529 /*target_bitrate=*/rate,
7530 /*stable_target_bitrate=*/rate,
7531 /*link_allocation=*/rate,
7532 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007533 /*rtt_ms=*/0,
7534 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007535
7536 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007537 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007538 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7539 frame.set_rotation(kVideoRotation_270);
7540 video_source_.IncomingCapturedFrame(frame);
7541 WaitForEncodedFrame(timestamp_ms);
7542 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7543
7544 // Set a higher target rate without changing the link_allocation. Should not
7545 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007546 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007548 /*target_bitrate=*/rate,
7549 /*stable_target_bitrate=*/new_stable_rate,
7550 /*link_allocation=*/rate,
7551 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007552 /*rtt_ms=*/0,
7553 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007554 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7555 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7556 video_stream_encoder_->Stop();
7557}
7558
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007559TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7560 test::ScopedFieldTrials field_trials(
7561 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7562 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7563 const int kFramerateFps = 30;
7564 const int kWidth = 1920;
7565 const int kHeight = 1080;
7566 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7567 // Works on screenshare mode.
7568 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7569 // We rely on the automatic resolution adaptation, but we handle framerate
7570 // adaptation manually by mocking the stats proxy.
7571 video_source_.set_adaptation_enabled(true);
7572
7573 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007575 DataRate::BitsPerSec(kTargetBitrateBps),
7576 DataRate::BitsPerSec(kTargetBitrateBps),
7577 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007578 video_stream_encoder_->SetSource(&video_source_,
7579 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007580 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007581
7582 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7583 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7584
7585 // Pass enough frames with the full update to trigger animation detection.
7586 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007587 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007588 frame.set_ntp_time_ms(timestamp_ms);
7589 frame.set_timestamp_us(timestamp_ms * 1000);
7590 video_source_.IncomingCapturedFrame(frame);
7591 WaitForEncodedFrame(timestamp_ms);
7592 }
7593
7594 // Resolution should be limited.
7595 rtc::VideoSinkWants expected;
7596 expected.max_framerate_fps = kFramerateFps;
7597 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007598 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007599
7600 // Pass one frame with no known update.
7601 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007602 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007603 frame.set_ntp_time_ms(timestamp_ms);
7604 frame.set_timestamp_us(timestamp_ms * 1000);
7605 frame.clear_update_rect();
7606
7607 video_source_.IncomingCapturedFrame(frame);
7608 WaitForEncodedFrame(timestamp_ms);
7609
7610 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007611 EXPECT_THAT(video_source_.sink_wants(),
7612 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007613
7614 video_stream_encoder_->Stop();
7615}
7616
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007617TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7618 const int kWidth = 720; // 540p adapted down.
7619 const int kHeight = 405;
7620 const int kNumFrames = 3;
7621 // Works on screenshare mode.
7622 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7623 /*num_spatial_layers=*/2, /*screenshare=*/true);
7624
7625 video_source_.set_adaptation_enabled(true);
7626
7627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7628 DataRate::BitsPerSec(kTargetBitrateBps),
7629 DataRate::BitsPerSec(kTargetBitrateBps),
7630 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7631
7632 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7633
7634 // Pass enough frames with the full update to trigger animation detection.
7635 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007636 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007637 frame.set_ntp_time_ms(timestamp_ms);
7638 frame.set_timestamp_us(timestamp_ms * 1000);
7639 video_source_.IncomingCapturedFrame(frame);
7640 WaitForEncodedFrame(timestamp_ms);
7641 }
7642
7643 video_stream_encoder_->Stop();
7644}
7645
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007646TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7647 const float downscale_factors[] = {4.0, 2.0, 1.0};
7648 const int number_layers =
7649 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7650 VideoEncoderConfig config;
7651 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7652 for (int i = 0; i < number_layers; ++i) {
7653 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7654 config.simulcast_layers[i].active = true;
7655 }
7656 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007657 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007658 "VP8", /*max qp*/ 56, /*screencast*/ false,
7659 /*screenshare enabled*/ false);
7660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7661 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7662 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7663 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7664
7665 // First initialization.
7666 // Encoder should be initialized. Next frame should be key frame.
7667 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7668 sink_.SetNumExpectedLayers(number_layers);
7669 int64_t timestamp_ms = kFrameIntervalMs;
7670 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7671 WaitForEncodedFrame(timestamp_ms);
7672 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7673 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7674 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7675 VideoFrameType::kVideoFrameKey,
7676 VideoFrameType::kVideoFrameKey}));
7677
7678 // Disable top layer.
7679 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7680 config.simulcast_layers[number_layers - 1].active = false;
7681 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7682 sink_.SetNumExpectedLayers(number_layers - 1);
7683 timestamp_ms += kFrameIntervalMs;
7684 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7685 WaitForEncodedFrame(timestamp_ms);
7686 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7687 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7688 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7689 VideoFrameType::kVideoFrameDelta,
7690 VideoFrameType::kVideoFrameDelta}));
7691
7692 // Re-enable top layer.
7693 // Encoder should be re-initialized. Next frame should be key frame.
7694 config.simulcast_layers[number_layers - 1].active = true;
7695 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7696 sink_.SetNumExpectedLayers(number_layers);
7697 timestamp_ms += kFrameIntervalMs;
7698 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7699 WaitForEncodedFrame(timestamp_ms);
7700 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7701 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7702 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7703 VideoFrameType::kVideoFrameKey,
7704 VideoFrameType::kVideoFrameKey}));
7705
7706 // Top layer max rate change.
7707 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7708 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7709 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7710 sink_.SetNumExpectedLayers(number_layers);
7711 timestamp_ms += kFrameIntervalMs;
7712 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7713 WaitForEncodedFrame(timestamp_ms);
7714 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7715 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7716 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7717 VideoFrameType::kVideoFrameDelta,
7718 VideoFrameType::kVideoFrameDelta}));
7719
7720 // Top layer resolution change.
7721 // Encoder should be re-initialized. Next frame should be key frame.
7722 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7723 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7724 sink_.SetNumExpectedLayers(number_layers);
7725 timestamp_ms += kFrameIntervalMs;
7726 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7727 WaitForEncodedFrame(timestamp_ms);
7728 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7729 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7730 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7731 VideoFrameType::kVideoFrameKey,
7732 VideoFrameType::kVideoFrameKey}));
7733 video_stream_encoder_->Stop();
7734}
7735
Henrik Boström1124ed12021-02-25 10:30:39 +01007736TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7737 const int kFrameWidth = 1280;
7738 const int kFrameHeight = 720;
7739
7740 SetUp();
7741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7742 DataRate::BitsPerSec(kTargetBitrateBps),
7743 DataRate::BitsPerSec(kTargetBitrateBps),
7744 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7745
7746 // Capturing a frame should reconfigure the encoder and expose the encoder
7747 // resolution, which is the same as the input frame.
7748 int64_t timestamp_ms = kFrameIntervalMs;
7749 video_source_.IncomingCapturedFrame(
7750 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7751 WaitForEncodedFrame(timestamp_ms);
7752 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7753 EXPECT_THAT(video_source_.sink_wants().resolutions,
7754 ::testing::ElementsAreArray(
7755 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7756
7757 video_stream_encoder_->Stop();
7758}
7759
7760TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7761 // Pick downscale factors such that we never encode at full resolution - this
7762 // is an interesting use case. The frame resolution influences the encoder
7763 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7764 // encoder should not ask for the frame resolution. This allows video frames
7765 // to have the appearence of one resolution but optimize its internal buffers
7766 // for what is actually encoded.
7767 const size_t kNumSimulcastLayers = 3u;
7768 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7769 const int kFrameWidth = 1280;
7770 const int kFrameHeight = 720;
7771 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7772 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7773 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7774 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7775 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7776 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7777
7778 VideoEncoderConfig config;
7779 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7780 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7781 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7782 config.simulcast_layers[i].active = true;
7783 }
7784 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007785 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007786 "VP8", /*max qp*/ 56, /*screencast*/ false,
7787 /*screenshare enabled*/ false);
7788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7789 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7790 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7791 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7792
7793 // Capture a frame with all layers active.
7794 int64_t timestamp_ms = kFrameIntervalMs;
7795 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7796 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7797 video_source_.IncomingCapturedFrame(
7798 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7799 WaitForEncodedFrame(timestamp_ms);
7800 // Expect encoded resolutions to match the expected simulcast layers.
7801 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7802 EXPECT_THAT(
7803 video_source_.sink_wants().resolutions,
7804 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7805
7806 // Capture a frame with one of the layers inactive.
7807 timestamp_ms += kFrameIntervalMs;
7808 config.simulcast_layers[2].active = false;
7809 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7810 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7811 video_source_.IncomingCapturedFrame(
7812 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7813 WaitForEncodedFrame(timestamp_ms);
7814
7815 // Expect encoded resolutions to match the expected simulcast layers.
7816 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7817 EXPECT_THAT(video_source_.sink_wants().resolutions,
7818 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7819
7820 // Capture a frame with all but one layer turned off.
7821 timestamp_ms += kFrameIntervalMs;
7822 config.simulcast_layers[1].active = false;
7823 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7824 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7825 video_source_.IncomingCapturedFrame(
7826 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7827 WaitForEncodedFrame(timestamp_ms);
7828
7829 // Expect encoded resolutions to match the expected simulcast layers.
7830 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7831 EXPECT_THAT(video_source_.sink_wants().resolutions,
7832 ::testing::ElementsAreArray({kLayer0Size}));
7833
7834 video_stream_encoder_->Stop();
7835}
7836
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007837TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
7838 // Enable encoder source to force encoder reconfig.
7839 encoder_factory_.SetHasInternalSource(true);
7840 ResetEncoder("VP8", 1, 1, 1, false);
7841
7842 // Set QP on encoded frame and pass the frame to encode complete callback.
7843 // Since QP is present QP parsing won't be triggered and the original value
7844 // should be kept.
7845 EncodedImage encoded_image;
7846 encoded_image.qp_ = 123;
7847 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7848 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7849 CodecSpecificInfo codec_info;
7850 codec_info.codecType = kVideoCodecVP8;
7851 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7852 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7853 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7854 video_stream_encoder_->Stop();
7855}
7856
7857TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
7858 // Enable encoder source to force encoder reconfig.
7859 encoder_factory_.SetHasInternalSource(true);
7860 ResetEncoder("VP8", 1, 1, 1, false);
7861
7862 // Pass an encoded frame without QP to encode complete callback. QP should be
7863 // parsed and set.
7864 EncodedImage encoded_image;
7865 encoded_image.qp_ = -1;
7866 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7867 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7868 CodecSpecificInfo codec_info;
7869 codec_info.codecType = kVideoCodecVP8;
7870 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7871 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7872 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7873 video_stream_encoder_->Stop();
7874}
7875
7876TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7877 webrtc::test::ScopedFieldTrials field_trials(
7878 "WebRTC-QpParsingKillSwitch/Enabled/");
7879
7880 // Enable encoder source to force encoder reconfig.
7881 encoder_factory_.SetHasInternalSource(true);
7882 ResetEncoder("VP8", 1, 1, 1, false);
7883
7884 EncodedImage encoded_image;
7885 encoded_image.qp_ = -1;
7886 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7887 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7888 CodecSpecificInfo codec_info;
7889 codec_info.codecType = kVideoCodecVP8;
7890 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7891 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7892 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7893 video_stream_encoder_->Stop();
7894}
7895
Sergey Silkind19e3b92021-03-16 10:05:30 +00007896TEST_F(VideoStreamEncoderTest,
7897 QualityScalingNotAllowed_QualityScalingDisabled) {
7898 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7899
7900 // Disable scaling settings in encoder info.
7901 fake_encoder_.SetQualityScaling(false);
7902 // Disable quality scaling in encoder config.
7903 video_encoder_config.is_quality_scaling_allowed = false;
7904 ConfigureEncoder(std::move(video_encoder_config));
7905
7906 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7907 DataRate::BitsPerSec(kTargetBitrateBps),
7908 DataRate::BitsPerSec(kTargetBitrateBps),
7909 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7910
7911 test::FrameForwarder source;
7912 video_stream_encoder_->SetSource(
7913 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7914 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7915 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7916
7917 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7918 WaitForEncodedFrame(1);
7919 video_stream_encoder_->TriggerQualityLow();
7920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7921
7922 video_stream_encoder_->Stop();
7923}
7924
7925#if !defined(WEBRTC_IOS)
7926// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
7927// disabled by default on iOS.
7928TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
7929 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7930
7931 // Disable scaling settings in encoder info.
7932 fake_encoder_.SetQualityScaling(false);
7933 // Enable quality scaling in encoder config.
7934 video_encoder_config.is_quality_scaling_allowed = true;
7935 ConfigureEncoder(std::move(video_encoder_config));
7936
7937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7938 DataRate::BitsPerSec(kTargetBitrateBps),
7939 DataRate::BitsPerSec(kTargetBitrateBps),
7940 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7941
7942 test::FrameForwarder source;
7943 video_stream_encoder_->SetSource(
7944 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7945 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7946 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7947
7948 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7949 WaitForEncodedFrame(1);
7950 video_stream_encoder_->TriggerQualityLow();
7951 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
7952
7953 video_stream_encoder_->Stop();
7954}
7955#endif
7956
Henrik Boström56db9ff2021-03-24 09:06:45 +01007957// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
7958class VideoStreamEncoderWithRealEncoderTest
7959 : public VideoStreamEncoderTest,
7960 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
7961 public:
7962 VideoStreamEncoderWithRealEncoderTest()
7963 : VideoStreamEncoderTest(),
7964 codec_type_(std::get<0>(GetParam())),
7965 allow_i420_conversion_(std::get<1>(GetParam())) {}
7966
7967 void SetUp() override {
7968 VideoStreamEncoderTest::SetUp();
7969 std::unique_ptr<VideoEncoder> encoder;
7970 switch (codec_type_) {
7971 case kVideoCodecVP8:
7972 encoder = VP8Encoder::Create();
7973 break;
7974 case kVideoCodecVP9:
7975 encoder = VP9Encoder::Create();
7976 break;
7977 case kVideoCodecAV1:
7978 encoder = CreateLibaomAv1Encoder();
7979 break;
7980 case kVideoCodecH264:
7981 encoder =
7982 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
7983 break;
7984 case kVideoCodecMultiplex:
7985 mock_encoder_factory_for_multiplex_ =
7986 std::make_unique<MockVideoEncoderFactory>();
7987 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
7988 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
7989 .WillRepeatedly([] { return VP8Encoder::Create(); });
7990 encoder = std::make_unique<MultiplexEncoderAdapter>(
7991 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
7992 false);
7993 break;
7994 default:
7995 RTC_NOTREACHED();
7996 }
7997 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
7998 }
7999
8000 void TearDown() override {
8001 video_stream_encoder_->Stop();
8002 // Ensure |video_stream_encoder_| is destroyed before
8003 // |encoder_proxy_factory_|.
8004 video_stream_encoder_.reset();
8005 VideoStreamEncoderTest::TearDown();
8006 }
8007
8008 protected:
8009 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8010 std::unique_ptr<VideoEncoder> encoder) {
8011 // Configure VSE to use the encoder.
8012 encoder_ = std::move(encoder);
8013 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8014 encoder_.get(), &encoder_selector_);
8015 video_send_config_.encoder_settings.encoder_factory =
8016 encoder_proxy_factory_.get();
8017 VideoEncoderConfig video_encoder_config;
8018 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8019 video_encoder_config_ = video_encoder_config.Copy();
8020 ConfigureEncoder(video_encoder_config_.Copy());
8021
8022 // Set bitrate to ensure frame is not dropped.
8023 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8024 DataRate::BitsPerSec(kTargetBitrateBps),
8025 DataRate::BitsPerSec(kTargetBitrateBps),
8026 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8027 }
8028
8029 const VideoCodecType codec_type_;
8030 const bool allow_i420_conversion_;
8031 NiceMock<MockEncoderSelector> encoder_selector_;
8032 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8033 std::unique_ptr<VideoEncoder> encoder_;
8034 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8035};
8036
8037TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8038 auto native_i420_frame = test::CreateMappableNativeFrame(
8039 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8040 video_source_.IncomingCapturedFrame(native_i420_frame);
8041 WaitForEncodedFrame(codec_width_, codec_height_);
8042
8043 auto mappable_native_buffer =
8044 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8045 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8046 mappable_native_buffer->GetMappedFramedBuffers();
8047 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8048 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8049 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8050 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8051}
8052
8053TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8054 auto native_nv12_frame = test::CreateMappableNativeFrame(
8055 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8056 video_source_.IncomingCapturedFrame(native_nv12_frame);
8057 WaitForEncodedFrame(codec_width_, codec_height_);
8058
8059 auto mappable_native_buffer =
8060 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8061 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8062 mappable_native_buffer->GetMappedFramedBuffers();
8063 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8064 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8065 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8066 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8067
8068 if (!allow_i420_conversion_) {
8069 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8070 }
8071}
8072
Erik Språng7444b192021-06-02 14:02:13 +02008073TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8074 if (codec_type_ == kVideoCodecMultiplex) {
8075 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8076 return;
8077 }
8078
8079 const size_t kNumSpatialLayers = 3u;
8080 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8081 const int kFrameWidth = 1280;
8082 const int kFrameHeight = 720;
8083 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8084 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8085 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8086 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8087 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8088 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8089
8090 VideoEncoderConfig config;
8091 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8092 test::FillEncoderConfiguration(codec_type_, 1, &config);
8093 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8094 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8095 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8096 vp9_settings.numberOfTemporalLayers = 3;
8097 vp9_settings.automaticResizeOn = false;
8098 config.encoder_specific_settings =
8099 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8100 vp9_settings);
8101 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8102 /*fps=*/30.0,
8103 /*first_active_layer=*/0,
8104 /*num_spatial_layers=*/3,
8105 /*num_temporal_layers=*/3,
8106 /*is_screenshare=*/false);
8107 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8108 test::FillEncoderConfiguration(codec_type_, 1, &config);
8109 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8110 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8111 /*fps=*/30.0,
8112 /*first_active_layer=*/0,
8113 /*num_spatial_layers=*/3,
8114 /*num_temporal_layers=*/3,
8115 /*is_screenshare=*/false);
8116 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8117 } else {
8118 // Simulcast for VP8/H264.
8119 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8120 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8121 config.simulcast_layers[i].scale_resolution_down_by =
8122 kDownscaleFactors[i];
8123 config.simulcast_layers[i].active = true;
8124 }
8125 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8126 // Turn off frame dropping to prevent flakiness.
8127 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8128 h264_settings.frameDroppingOn = false;
8129 config.encoder_specific_settings = rtc::make_ref_counted<
8130 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8131 }
8132 }
8133
8134 auto set_layer_active = [&](int layer_idx, bool active) {
8135 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8136 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8137 config.spatial_layers[layer_idx].active = active;
8138 } else {
8139 config.simulcast_layers[layer_idx].active = active;
8140 }
8141 };
8142
8143 config.video_stream_factory =
8144 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8145 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8146 /*screencast*/ false,
8147 /*screenshare enabled*/ false);
8148 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8149 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8150 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8151 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8152
8153 // Capture a frame with all layers active.
8154 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8155 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8156 int64_t timestamp_ms = kFrameIntervalMs;
8157 video_source_.IncomingCapturedFrame(
8158 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8159
8160 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8161 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8162
8163 // Capture a frame with one of the layers inactive.
8164 set_layer_active(2, false);
8165 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8166 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8167 timestamp_ms += kFrameIntervalMs;
8168 video_source_.IncomingCapturedFrame(
8169 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8170 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8171
8172 // New target bitrates signaled based on lower resolution.
8173 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8175 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8176 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8177
8178 // Re-enable the top layer.
8179 set_layer_active(2, true);
8180 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8181 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8182 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8183
8184 // Bitrate target adjusted back up to enable HD layer...
8185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8186 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8187 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8188 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8189
8190 // ...then add a new frame.
8191 timestamp_ms += kFrameIntervalMs;
8192 video_source_.IncomingCapturedFrame(
8193 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8194 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8195 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8196
8197 video_stream_encoder_->Stop();
8198}
8199
Henrik Boström56db9ff2021-03-24 09:06:45 +01008200std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8201 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8202 VideoCodecType codec_type = std::get<0>(info.param);
8203 bool allow_i420_conversion = std::get<1>(info.param);
8204 std::string str;
8205 switch (codec_type) {
8206 case kVideoCodecGeneric:
8207 str = "Generic";
8208 break;
8209 case kVideoCodecVP8:
8210 str = "VP8";
8211 break;
8212 case kVideoCodecVP9:
8213 str = "VP9";
8214 break;
8215 case kVideoCodecAV1:
8216 str = "AV1";
8217 break;
8218 case kVideoCodecH264:
8219 str = "H264";
8220 break;
8221 case kVideoCodecMultiplex:
8222 str = "Multiplex";
8223 break;
8224 default:
8225 RTC_NOTREACHED();
8226 }
8227 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8228 return str;
8229}
8230
8231constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8232 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8233constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8234 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8235constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8236 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8237constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8238 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8239#if defined(WEBRTC_USE_H264)
8240constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8241 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8242
8243// The windows compiler does not tolerate #if statements inside the
8244// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8245// and without H264).
8246INSTANTIATE_TEST_SUITE_P(
8247 All,
8248 VideoStreamEncoderWithRealEncoderTest,
8249 ::testing::Values(kVP8DisallowConversion,
8250 kVP9DisallowConversion,
8251 kAV1AllowConversion,
8252 kMultiplexDisallowConversion,
8253 kH264AllowConversion),
8254 TestParametersVideoCodecAndAllowI420ConversionToString);
8255#else
8256INSTANTIATE_TEST_SUITE_P(
8257 All,
8258 VideoStreamEncoderWithRealEncoderTest,
8259 ::testing::Values(kVP8DisallowConversion,
8260 kVP9DisallowConversion,
8261 kAV1AllowConversion,
8262 kMultiplexDisallowConversion),
8263 TestParametersVideoCodecAndAllowI420ConversionToString);
8264#endif
8265
perkj26091b12016-09-01 01:17:40 -07008266} // namespace webrtc