blob: d64341901e8b93b836f9dcb5c2f780a0da4c0d26 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020045#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020046#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010047#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020048#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010049#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020050#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020051#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080052#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020053#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010054#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020055#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "test/encoder_settings.h"
57#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020058#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010059#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/gmock.h"
61#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010062#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020063#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020064#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070066
67namespace webrtc {
68
sprang57c2fff2017-01-16 06:24:02 -080069using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020070using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020071using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020072using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Ge;
75using ::testing::Gt;
76using ::testing::Le;
77using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010078using ::testing::Matcher;
79using ::testing::NiceMock;
80using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010081using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080083
perkj803d97f2016-11-01 11:45:46 -070084namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020085const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010086const int kQpLow = 1;
87const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020088const int kMinFramerateFps = 2;
89const int kMinBalancedFramerateFps = 7;
90const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080091const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010092const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020093const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010094const uint32_t kSimulcastTargetBitrateBps = 3150000;
95const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080096const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070097const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020099const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200100const VideoEncoder::ResolutionBitrateLimits
101 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
102const VideoEncoder::ResolutionBitrateLimits
103 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800104
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200105uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
106 0x00, 0x00, 0x03, 0x03, 0xF4,
107 0x05, 0x03, 0xC7, 0xE0, 0x1B,
108 0x41, 0x10, 0x8D, 0x00};
109
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100110const uint8_t kCodedFrameVp8Qp25[] = {
111 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
112 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
113 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
114
perkj803d97f2016-11-01 11:45:46 -0700115class TestBuffer : public webrtc::I420Buffer {
116 public:
117 TestBuffer(rtc::Event* event, int width, int height)
118 : I420Buffer(width, height), event_(event) {}
119
120 private:
121 friend class rtc::RefCountedObject<TestBuffer>;
122 ~TestBuffer() override {
123 if (event_)
124 event_->Set();
125 }
126 rtc::Event* const event_;
127};
128
Henrik Boström56db9ff2021-03-24 09:06:45 +0100129// A fake native buffer that can't be converted to I420. Upon scaling, it
130// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700131class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
132 public:
133 FakeNativeBuffer(rtc::Event* event, int width, int height)
134 : event_(event), width_(width), height_(height) {}
135 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
136 int width() const override { return width_; }
137 int height() const override { return height_; }
138 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
139 return nullptr;
140 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100141 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
142 int offset_x,
143 int offset_y,
144 int crop_width,
145 int crop_height,
146 int scaled_width,
147 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200148 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
149 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100150 }
Noah Richards51db4212019-06-12 06:59:12 -0700151
152 private:
153 friend class rtc::RefCountedObject<FakeNativeBuffer>;
154 ~FakeNativeBuffer() override {
155 if (event_)
156 event_->Set();
157 }
158 rtc::Event* const event_;
159 const int width_;
160 const int height_;
161};
162
Evan Shrubsole895556e2020-10-05 09:15:13 +0200163// A fake native buffer that is backed by an NV12 buffer.
164class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
165 public:
166 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
167 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
168
169 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
170 int width() const override { return nv12_buffer_->width(); }
171 int height() const override { return nv12_buffer_->height(); }
172 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
173 return nv12_buffer_->ToI420();
174 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200175 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
176 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
177 if (absl::c_find(types, Type::kNV12) != types.end()) {
178 return nv12_buffer_;
179 }
180 return nullptr;
181 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200182 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
183
184 private:
185 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
186 ~FakeNV12NativeBuffer() override {
187 if (event_)
188 event_->Set();
189 }
190 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
191 rtc::Event* const event_;
192};
193
Niels Möller7dc26b72017-12-06 10:27:48 +0100194class CpuOveruseDetectorProxy : public OveruseFrameDetector {
195 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200196 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
197 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200198 last_target_framerate_fps_(-1),
199 framerate_updated_event_(true /* manual_reset */,
200 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100201 virtual ~CpuOveruseDetectorProxy() {}
202
203 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200204 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 last_target_framerate_fps_ = framerate_fps;
206 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200207 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100208 }
209
210 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200211 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 return last_target_framerate_fps_;
213 }
214
Niels Möller4db138e2018-04-19 09:04:13 +0200215 CpuOveruseOptions GetOptions() { return options_; }
216
Henrik Boström381d1092020-05-12 18:49:07 +0200217 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219 private:
Markus Handella3765182020-07-08 13:13:32 +0200220 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100221 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200222 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100223};
224
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200225class FakeVideoSourceRestrictionsListener
226 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200227 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200228 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200229 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200230 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 RTC_DCHECK(was_restrictions_updated_);
232 }
233
234 rtc::Event* restrictions_updated_event() {
235 return &restrictions_updated_event_;
236 }
237
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200238 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200239 void OnVideoSourceRestrictionsUpdated(
240 VideoSourceRestrictions restrictions,
241 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200242 rtc::scoped_refptr<Resource> reason,
243 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200244 was_restrictions_updated_ = true;
245 restrictions_updated_event_.Set();
246 }
247
248 private:
249 bool was_restrictions_updated_;
250 rtc::Event restrictions_updated_event_;
251};
252
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200253auto WantsFps(Matcher<int> fps_matcher) {
254 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
255 fps_matcher);
256}
257
258auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
259 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
260 AllOf(max_pixel_matcher, Gt(0)));
261}
262
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200263auto ResolutionMax() {
264 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200265 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200266 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
267 Eq(absl::nullopt)));
268}
269
270auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272}
273
274auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMax() {
287 return AllOf(FpsMax(), ResolutionMax());
288}
289
290auto UnlimitedSinkWants() {
291 return AllOf(FpsUnlimited(), ResolutionMax());
292}
293
294auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
295 Matcher<int> fps_range_matcher;
296
297 if (last_frame_pixels <= 320 * 240) {
298 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200299 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300 fps_range_matcher = AllOf(Ge(10), Le(15));
301 } else if (last_frame_pixels <= 640 * 480) {
302 fps_range_matcher = Ge(15);
303 } else {
304 fps_range_matcher = Eq(kDefaultFramerate);
305 }
306 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
307 fps_range_matcher);
308}
309
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200310auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
311 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
312 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
313}
314
315auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
316 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
321}
322
323auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
325 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
326}
327
328auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
329 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
330 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
331}
332
333auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
334 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
335 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
336}
337
338auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
341}
342
mflodmancc3d4422017-08-03 08:27:51 -0700343class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700344 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200345 VideoStreamEncoderUnderTest(TimeController* time_controller,
346 TaskQueueFactory* task_queue_factory,
347 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100348 const VideoStreamEncoderSettings& settings,
349 VideoStreamEncoder::BitrateAllocationCallbackType
350 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200351 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100352 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200353 stats_proxy,
354 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200355 std::unique_ptr<OveruseFrameDetector>(
356 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100357 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100358 task_queue_factory,
359 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200360 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200361 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200362 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200363 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200364 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200365 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200366 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200367 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100368 }
perkj803d97f2016-11-01 11:45:46 -0700369
Henrik Boström381d1092020-05-12 18:49:07 +0200370 void SetSourceAndWaitForRestrictionsUpdated(
371 rtc::VideoSourceInterface<VideoFrame>* source,
372 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200373 FakeVideoSourceRestrictionsListener listener;
374 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200375 SetSource(source, degradation_preference);
376 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200377 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200378 }
379
380 void SetSourceAndWaitForFramerateUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
383 overuse_detector_proxy_->framerate_updated_event()->Reset();
384 SetSource(source, degradation_preference);
385 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
386 }
387
388 void OnBitrateUpdatedAndWaitForManagedResources(
389 DataRate target_bitrate,
390 DataRate stable_target_bitrate,
391 DataRate link_allocation,
392 uint8_t fraction_lost,
393 int64_t round_trip_time_ms,
394 double cwnd_reduce_ratio) {
395 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
396 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
397 // Bitrate is updated on the encoder queue.
398 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200399 }
400
kthelgason2fc52542017-03-03 00:24:41 -0800401 // This is used as a synchronisation mechanism, to make sure that the
402 // encoder queue is not blocked before we start sending it frames.
403 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100404 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200405 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800406 ASSERT_TRUE(event.Wait(5000));
407 }
408
Henrik Boström91aa7322020-04-28 12:24:33 +0200409 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200410 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200411 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200412 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 event.Set();
415 });
416 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200417 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200418 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200419
Henrik Boström91aa7322020-04-28 12:24:33 +0200420 void TriggerCpuUnderuse() {
421 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200422 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200423 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200424 event.Set();
425 });
426 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200427 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200428 }
kthelgason876222f2016-11-29 01:44:11 -0800429
Henrik Boström91aa7322020-04-28 12:24:33 +0200430 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200431 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200433 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200434 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200435 event.Set();
436 });
437 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200438 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200440 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200442 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200443 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 event.Set();
445 });
446 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200447 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 }
449
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100451 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200452 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
453 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700455};
456
Noah Richards51db4212019-06-12 06:59:12 -0700457// Simulates simulcast behavior and makes highest stream resolutions divisible
458// by 4.
459class CroppingVideoStreamFactory
460 : public VideoEncoderConfig::VideoStreamFactoryInterface {
461 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200462 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700463
464 private:
465 std::vector<VideoStream> CreateEncoderStreams(
466 int width,
467 int height,
468 const VideoEncoderConfig& encoder_config) override {
469 std::vector<VideoStream> streams = test::CreateVideoStreams(
470 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700471 return streams;
472 }
Noah Richards51db4212019-06-12 06:59:12 -0700473};
474
sprangb1ca0732017-02-01 08:38:12 -0800475class AdaptingFrameForwarder : public test::FrameForwarder {
476 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 explicit AdaptingFrameForwarder(TimeController* time_controller)
478 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700479 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800480
481 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200482 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800483 adaptation_enabled_ = enabled;
484 }
485
asaperssonfab67072017-04-04 05:51:49 -0700486 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800488 return adaptation_enabled_;
489 }
490
Henrik Boström1124ed12021-02-25 10:30:39 +0100491 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
492 // the resolution or frame rate was different than it is currently. If
493 // something else is modified, such as encoder resolutions, but the resolution
494 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700495 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200496 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700497 return last_wants_;
498 }
499
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200500 absl::optional<int> last_sent_width() const { return last_width_; }
501 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800502
sprangb1ca0732017-02-01 08:38:12 -0800503 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200504 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
505 time_controller_->AdvanceTime(TimeDelta::Millis(0));
506
sprangb1ca0732017-02-01 08:38:12 -0800507 int cropped_width = 0;
508 int cropped_height = 0;
509 int out_width = 0;
510 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700511 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
513 << "w=" << video_frame.width()
514 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700515 if (adapter_.AdaptFrameResolution(
516 video_frame.width(), video_frame.height(),
517 video_frame.timestamp_us() * 1000, &cropped_width,
518 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100519 VideoFrame adapted_frame =
520 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200521 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100522 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200523 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 .set_timestamp_ms(99)
525 .set_rotation(kVideoRotation_0)
526 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-02 23:53:04 -0700534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700540 }
sprangb1ca0732017-02-01 08:38:12 -0800541 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800546 }
547 }
548
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200549 void OnOutputFormatRequest(int width, int height) {
550 absl::optional<std::pair<int, int>> target_aspect_ratio =
551 std::make_pair(width, height);
552 absl::optional<int> max_pixel_count = width * height;
553 absl::optional<int> max_fps;
554 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
555 max_fps);
556 }
557
sprangb1ca0732017-02-01 08:38:12 -0800558 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
559 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200560 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100561 rtc::VideoSinkWants prev_wants = sink_wants_locked();
562 bool did_adapt =
563 prev_wants.max_pixel_count != wants.max_pixel_count ||
564 prev_wants.target_pixel_count != wants.target_pixel_count ||
565 prev_wants.max_framerate_fps != wants.max_framerate_fps;
566 if (did_adapt) {
567 last_wants_ = prev_wants;
568 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100569 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200570 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800571 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200572
573 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800574 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200575 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
576 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200577 absl::optional<int> last_width_;
578 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800579};
sprangc5d62e22017-04-02 23:53:04 -0700580
Niels Möller213618e2018-07-24 09:29:58 +0200581// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700582class MockableSendStatisticsProxy : public SendStatisticsProxy {
583 public:
584 MockableSendStatisticsProxy(Clock* clock,
585 const VideoSendStream::Config& config,
586 VideoEncoderConfig::ContentType content_type)
587 : SendStatisticsProxy(clock, config, content_type) {}
588
589 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200590 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700591 if (mock_stats_)
592 return *mock_stats_;
593 return SendStatisticsProxy::GetStats();
594 }
595
Niels Möller213618e2018-07-24 09:29:58 +0200596 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200597 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200598 if (mock_stats_)
599 return mock_stats_->input_frame_rate;
600 return SendStatisticsProxy::GetInputFrameRate();
601 }
sprangc5d62e22017-04-02 23:53:04 -0700602 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200603 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700604 mock_stats_.emplace(stats);
605 }
606
607 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200608 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700609 mock_stats_.reset();
610 }
611
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200612 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
613 on_frame_dropped_ = std::move(callback);
614 }
615
sprangc5d62e22017-04-02 23:53:04 -0700616 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 void OnFrameDropped(DropReason reason) override {
618 SendStatisticsProxy::OnFrameDropped(reason);
619 if (on_frame_dropped_)
620 on_frame_dropped_(reason);
621 }
622
Markus Handella3765182020-07-08 13:13:32 +0200623 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200624 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200625 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700626};
627
philipel9b058032020-02-10 11:30:00 +0100628class MockEncoderSelector
629 : public VideoEncoderFactory::EncoderSelectorInterface {
630 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200631 MOCK_METHOD(void,
632 OnCurrentEncoder,
633 (const SdpVideoFormat& format),
634 (override));
635 MOCK_METHOD(absl::optional<SdpVideoFormat>,
636 OnAvailableBitrate,
637 (const DataRate& rate),
638 (override));
639 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100640};
641
perkj803d97f2016-11-01 11:45:46 -0700642} // namespace
643
mflodmancc3d4422017-08-03 08:27:51 -0700644class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700645 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200646 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700647
mflodmancc3d4422017-08-03 08:27:51 -0700648 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700649 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700650 codec_width_(320),
651 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200652 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200654 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700655 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200656 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700657 video_send_config_,
658 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200659 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700660
661 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700662 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700663 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200664 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800665 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200666 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200667 video_send_config_.rtp.payload_name = "FAKE";
668 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700669
Per512ecb32016-09-23 15:52:06 +0200670 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200671 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200672 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
673 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
674 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100675 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700676
Niels Möllerf1338562018-04-26 09:51:47 +0200677 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800678 }
679
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100680 void ConfigureEncoder(
681 VideoEncoderConfig video_encoder_config,
682 VideoStreamEncoder::BitrateAllocationCallbackType
683 allocation_callback_type =
684 VideoStreamEncoder::BitrateAllocationCallbackType::
685 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700686 if (video_stream_encoder_)
687 video_stream_encoder_->Stop();
688 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200689 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100690 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700691 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700693 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700694 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
695 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200696 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700697 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800698 }
699
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100700 void ResetEncoder(const std::string& payload_name,
701 size_t num_streams,
702 size_t num_temporal_layers,
703 unsigned char num_spatial_layers,
704 bool screenshare,
705 VideoStreamEncoder::BitrateAllocationCallbackType
706 allocation_callback_type =
707 VideoStreamEncoder::BitrateAllocationCallbackType::
708 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200709 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800710
711 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200712 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
713 num_streams, &video_encoder_config);
714 for (auto& layer : video_encoder_config.simulcast_layers) {
715 layer.num_temporal_layers = num_temporal_layers;
716 layer.max_framerate = kDefaultFramerate;
717 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100718 video_encoder_config.max_bitrate_bps =
719 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700720 video_encoder_config.content_type =
721 screenshare ? VideoEncoderConfig::ContentType::kScreen
722 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700723 if (payload_name == "VP9") {
724 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
725 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200726 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200728 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
729 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700730 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100731 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700732 }
733
sprang57c2fff2017-01-16 06:24:02 -0800734 VideoFrame CreateFrame(int64_t ntp_time_ms,
735 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200736 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200737 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200738 destruction_event, codec_width_, codec_height_))
739 .set_ntp_time_ms(ntp_time_ms)
740 .set_timestamp_ms(99)
741 .set_rotation(kVideoRotation_0)
742 .build();
perkj26091b12016-09-01 01:17:40 -0700743 }
744
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100745 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
746 rtc::Event* destruction_event,
747 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200748 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200749 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200750 destruction_event, codec_width_, codec_height_))
751 .set_ntp_time_ms(ntp_time_ms)
752 .set_timestamp_ms(99)
753 .set_rotation(kVideoRotation_0)
754 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
755 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100756 }
757
sprang57c2fff2017-01-16 06:24:02 -0800758 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200759 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
760 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200761 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200762 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200763 .set_ntp_time_ms(ntp_time_ms)
764 .set_timestamp_ms(ntp_time_ms)
765 .set_rotation(kVideoRotation_0)
766 .build();
perkj803d97f2016-11-01 11:45:46 -0700767 }
768
Evan Shrubsole895556e2020-10-05 09:15:13 +0200769 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200770 return VideoFrame::Builder()
771 .set_video_frame_buffer(NV12Buffer::Create(width, height))
772 .set_ntp_time_ms(ntp_time_ms)
773 .set_timestamp_ms(ntp_time_ms)
774 .set_rotation(kVideoRotation_0)
775 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200776 }
777
Noah Richards51db4212019-06-12 06:59:12 -0700778 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
779 rtc::Event* destruction_event,
780 int width,
781 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200782 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200783 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200784 destruction_event, width, height))
785 .set_ntp_time_ms(ntp_time_ms)
786 .set_timestamp_ms(99)
787 .set_rotation(kVideoRotation_0)
788 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700789 }
790
Evan Shrubsole895556e2020-10-05 09:15:13 +0200791 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
792 rtc::Event* destruction_event,
793 int width,
794 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200795 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200796 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200797 destruction_event, width, height))
798 .set_ntp_time_ms(ntp_time_ms)
799 .set_timestamp_ms(99)
800 .set_rotation(kVideoRotation_0)
801 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200802 }
803
Noah Richards51db4212019-06-12 06:59:12 -0700804 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
805 rtc::Event* destruction_event) const {
806 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
807 codec_height_);
808 }
809
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100810 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100812 DataRate::BitsPerSec(kTargetBitrateBps),
813 DataRate::BitsPerSec(kTargetBitrateBps),
814 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100815
816 video_source_.IncomingCapturedFrame(
817 CreateFrame(1, codec_width_, codec_height_));
818 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200819 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100820 }
821
sprang4847ae62017-06-27 07:06:52 -0700822 void WaitForEncodedFrame(int64_t expected_ntp_time) {
823 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700825 }
826
827 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
828 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200829 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700830 return ok;
831 }
832
833 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
834 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700836 }
837
838 void ExpectDroppedFrame() {
839 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200840 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700841 }
842
843 bool WaitForFrame(int64_t timeout_ms) {
844 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200845 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700846 return ok;
847 }
848
perkj26091b12016-09-01 01:17:40 -0700849 class TestEncoder : public test::FakeEncoder {
850 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200851 explicit TestEncoder(TimeController* time_controller)
852 : FakeEncoder(time_controller->GetClock()),
853 time_controller_(time_controller) {
854 RTC_DCHECK(time_controller_);
855 }
perkj26091b12016-09-01 01:17:40 -0700856
asaperssonfab67072017-04-04 05:51:49 -0700857 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200858 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700859 return config_;
860 }
861
862 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200863 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700864 block_next_encode_ = true;
865 }
866
Erik Språngaed30702018-11-05 12:57:17 +0100867 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200868 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200869 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100870 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100871 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100872 info.scaling_settings = VideoEncoder::ScalingSettings(
873 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100874 }
875 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100876 for (int i = 0; i < kMaxSpatialLayers; ++i) {
877 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100878 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100879 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100880 for (int tid = 0; tid < num_layers; ++tid)
881 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100882 }
883 }
Erik Språngaed30702018-11-05 12:57:17 +0100884 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200885
886 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100887 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200888 info.apply_alignment_to_all_simulcast_layers =
889 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200890 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800891 if (is_qp_trusted_.has_value()) {
892 info.is_qp_trusted = is_qp_trusted_;
893 }
Erik Språngaed30702018-11-05 12:57:17 +0100894 return info;
kthelgason876222f2016-11-29 01:44:11 -0800895 }
896
Erik Språngb7cb7b52019-02-26 15:52:33 +0100897 int32_t RegisterEncodeCompleteCallback(
898 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200899 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100900 encoded_image_callback_ = callback;
901 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
902 }
903
perkjfa10b552016-10-02 23:45:26 -0700904 void ContinueEncode() { continue_encode_event_.Set(); }
905
906 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
907 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200908 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700909 EXPECT_EQ(timestamp_, timestamp);
910 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
911 }
912
kthelgason2fc52542017-03-03 00:24:41 -0800913 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200914 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800915 quality_scaling_ = b;
916 }
kthelgasonad9010c2017-02-14 00:46:51 -0800917
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100918 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200919 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100920 requested_resolution_alignment_ = requested_resolution_alignment;
921 }
922
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200923 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
924 MutexLock lock(&local_mutex_);
925 apply_alignment_to_all_simulcast_layers_ = b;
926 }
927
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100928 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200929 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100930 is_hardware_accelerated_ = is_hardware_accelerated;
931 }
932
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100933 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
934 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200935 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100936 temporal_layers_supported_[spatial_idx] = supported;
937 }
938
Sergey Silkin6456e352019-07-08 17:56:40 +0200939 void SetResolutionBitrateLimits(
940 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200941 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200942 resolution_bitrate_limits_ = thresholds;
943 }
944
sprangfe627f32017-03-29 08:24:59 -0700945 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200946 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700947 force_init_encode_failed_ = force_failure;
948 }
949
Niels Möller6bb5ab92019-01-11 11:11:10 +0100950 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200951 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100952 rate_factor_ = rate_factor;
953 }
954
Erik Språngd7329ca2019-02-21 21:19:53 +0100955 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200956 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100957 return last_framerate_;
958 }
959
Erik Språngd7329ca2019-02-21 21:19:53 +0100960 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200961 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100962 return last_update_rect_;
963 }
964
Niels Möller87e2d782019-03-07 10:18:23 +0100965 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200966 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100967 return last_frame_types_;
968 }
969
970 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100971 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100972 keyframe ? VideoFrameType::kVideoFrameKey
973 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100974 {
Markus Handella3765182020-07-08 13:13:32 +0200975 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100976 last_frame_types_ = frame_type;
977 }
Niels Möllerb859b322019-03-07 12:40:01 +0100978 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100979 }
980
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100981 void InjectEncodedImage(const EncodedImage& image,
982 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200983 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100984 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100985 }
986
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200987 void SetEncodedImageData(
988 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200989 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200990 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200991 }
992
Erik Språngd7329ca2019-02-21 21:19:53 +0100993 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200994 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100995 expect_null_frame_ = true;
996 }
997
Erik Språng5056af02019-09-02 15:53:11 +0200998 absl::optional<VideoEncoder::RateControlParameters>
999 GetAndResetLastRateControlSettings() {
1000 auto settings = last_rate_control_settings_;
1001 last_rate_control_settings_.reset();
1002 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001003 }
1004
Henrik Boström56db9ff2021-03-24 09:06:45 +01001005 int GetLastInputWidth() const {
1006 MutexLock lock(&local_mutex_);
1007 return last_input_width_;
1008 }
1009
1010 int GetLastInputHeight() const {
1011 MutexLock lock(&local_mutex_);
1012 return last_input_height_;
1013 }
1014
Evan Shrubsole895556e2020-10-05 09:15:13 +02001015 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1016 MutexLock lock(&local_mutex_);
1017 return last_input_pixel_format_;
1018 }
1019
Sergey Silkin5ee69672019-07-02 14:18:34 +02001020 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +02001021 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001022 return num_encoder_initializations_;
1023 }
1024
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001025 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001026 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001027 return num_set_rates_;
1028 }
1029
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001030 VideoCodec video_codec() const {
1031 MutexLock lock(&local_mutex_);
1032 return video_codec_;
1033 }
1034
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001035 void SetPreferredPixelFormats(
1036 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1037 pixel_formats) {
1038 MutexLock lock(&local_mutex_);
1039 preferred_pixel_formats_ = std::move(pixel_formats);
1040 }
1041
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001042 void SetIsQpTrusted(absl::optional<bool> trusted) {
1043 MutexLock lock(&local_mutex_);
1044 is_qp_trusted_ = trusted;
1045 }
1046
perkjfa10b552016-10-02 23:45:26 -07001047 private:
perkj26091b12016-09-01 01:17:40 -07001048 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001049 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001050 bool block_encode;
1051 {
Markus Handella3765182020-07-08 13:13:32 +02001052 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001053 if (expect_null_frame_) {
1054 EXPECT_EQ(input_image.timestamp(), 0u);
1055 EXPECT_EQ(input_image.width(), 1);
1056 last_frame_types_ = *frame_types;
1057 expect_null_frame_ = false;
1058 } else {
1059 EXPECT_GT(input_image.timestamp(), timestamp_);
1060 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1061 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1062 }
perkj26091b12016-09-01 01:17:40 -07001063
1064 timestamp_ = input_image.timestamp();
1065 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001066 last_input_width_ = input_image.width();
1067 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001068 block_encode = block_next_encode_;
1069 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001070 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001071 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001072 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001073 }
Niels Möllerb859b322019-03-07 12:40:01 +01001074 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001075 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001076 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001077
perkj26091b12016-09-01 01:17:40 -07001078 return result;
1079 }
1080
Niels Möller08ae7ce2020-09-23 15:58:12 +02001081 CodecSpecificInfo EncodeHook(
1082 EncodedImage& encoded_image,
1083 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001084 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001085 {
1086 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001087 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001088 }
1089 MutexLock lock(&local_mutex_);
1090 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001091 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001092 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001093 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001094 }
1095
sprangfe627f32017-03-29 08:24:59 -07001096 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001097 const Settings& settings) override {
1098 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001099
Markus Handella3765182020-07-08 13:13:32 +02001100 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001101 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001102
1103 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001104 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001105
Erik Språng82fad3d2018-03-21 09:57:23 +01001106 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001107 // Simulate setting up temporal layers, in order to validate the life
1108 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001109 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001110 frame_buffer_controller_ =
1111 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001112 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001113 if (force_init_encode_failed_) {
1114 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001115 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001116 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001117
Erik Språngb7cb7b52019-02-26 15:52:33 +01001118 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001119 return res;
1120 }
1121
Erik Språngb7cb7b52019-02-26 15:52:33 +01001122 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001123 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001124 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1125 initialized_ = EncoderState::kUninitialized;
1126 return FakeEncoder::Release();
1127 }
1128
Erik Språng16cb8f52019-04-12 13:59:09 +02001129 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001130 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001131 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001132 VideoBitrateAllocation adjusted_rate_allocation;
1133 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1134 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001135 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001136 adjusted_rate_allocation.SetBitrate(
1137 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001138 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001139 rate_factor_));
1140 }
1141 }
1142 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001143 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001144 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001145 RateControlParameters adjusted_paramters = parameters;
1146 adjusted_paramters.bitrate = adjusted_rate_allocation;
1147 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001148 }
1149
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001150 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001151 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001152 enum class EncoderState {
1153 kUninitialized,
1154 kInitializationFailed,
1155 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001156 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1157 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001158 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001159 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1160 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1161 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1162 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1163 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1164 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001165 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1166 false;
Markus Handella3765182020-07-08 13:13:32 +02001167 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001168 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1169 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001170 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001171 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001172 absl::optional<bool>
1173 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001174 local_mutex_);
1175 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1176 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1177 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001178 absl::optional<VideoEncoder::RateControlParameters>
1179 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001180 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1181 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001182 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001183 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001184 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1185 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001186 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001187 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001188 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001189 RTC_GUARDED_BY(local_mutex_);
1190 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001191 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001192 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1193 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001194 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1195 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001196 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001197 };
1198
mflodmancc3d4422017-08-03 08:27:51 -07001199 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001200 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001201 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1202 : time_controller_(time_controller), test_encoder_(test_encoder) {
1203 RTC_DCHECK(time_controller_);
1204 }
perkj26091b12016-09-01 01:17:40 -07001205
perkj26091b12016-09-01 01:17:40 -07001206 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001207 EXPECT_TRUE(
1208 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1209 }
1210
1211 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1212 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001213 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001214 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001215 return false;
perkj26091b12016-09-01 01:17:40 -07001216 {
Markus Handella3765182020-07-08 13:13:32 +02001217 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001218 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001219 }
1220 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001221 return true;
perkj26091b12016-09-01 01:17:40 -07001222 }
1223
sprangb1ca0732017-02-01 08:38:12 -08001224 void WaitForEncodedFrame(uint32_t expected_width,
1225 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001226 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001227 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001228 }
1229
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001230 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001231 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001232 uint32_t width = 0;
1233 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001234 {
Markus Handella3765182020-07-08 13:13:32 +02001235 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001236 width = last_width_;
1237 height = last_height_;
1238 }
1239 EXPECT_EQ(expected_height, height);
1240 EXPECT_EQ(expected_width, width);
1241 }
1242
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001243 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1244 VideoRotation rotation;
1245 {
Markus Handella3765182020-07-08 13:13:32 +02001246 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001247 rotation = last_rotation_;
1248 }
1249 EXPECT_EQ(expected_rotation, rotation);
1250 }
1251
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001252 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001253
sprangc5d62e22017-04-02 23:53:04 -07001254 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001255 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1256 bool ret = encoded_frame_event_.Wait(timeout_ms);
1257 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1258 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001259 }
1260
perkj26091b12016-09-01 01:17:40 -07001261 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001262 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001263 expect_frames_ = false;
1264 }
1265
asaperssonfab67072017-04-04 05:51:49 -07001266 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001267 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001268 return number_of_reconfigurations_;
1269 }
1270
asaperssonfab67072017-04-04 05:51:49 -07001271 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001272 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001273 return min_transmit_bitrate_bps_;
1274 }
1275
Erik Språngd7329ca2019-02-21 21:19:53 +01001276 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001277 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001278 num_expected_layers_ = num_layers;
1279 }
1280
Erik Språngb7cb7b52019-02-26 15:52:33 +01001281 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001282 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 return last_capture_time_ms_;
1284 }
1285
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001286 const EncodedImage& GetLastEncodedImage() {
1287 MutexLock lock(&mutex_);
1288 return last_encoded_image_;
1289 }
1290
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001291 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001292 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001293 return std::move(last_encoded_image_data_);
1294 }
1295
Per Kjellanderdcef6412020-10-07 15:09:05 +02001296 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1297 MutexLock lock(&mutex_);
1298 return last_bitrate_allocation_;
1299 }
1300
1301 int number_of_bitrate_allocations() const {
1302 MutexLock lock(&mutex_);
1303 return number_of_bitrate_allocations_;
1304 }
1305
Per Kjellandera9434842020-10-15 17:53:22 +02001306 VideoLayersAllocation GetLastVideoLayersAllocation() {
1307 MutexLock lock(&mutex_);
1308 return last_layers_allocation_;
1309 }
1310
1311 int number_of_layers_allocations() const {
1312 MutexLock lock(&mutex_);
1313 return number_of_layers_allocations_;
1314 }
1315
perkj26091b12016-09-01 01:17:40 -07001316 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001317 Result OnEncodedImage(
1318 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001319 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001320 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001321 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001322 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001323 last_encoded_image_data_ = std::vector<uint8_t>(
1324 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001325 uint32_t timestamp = encoded_image.Timestamp();
1326 if (last_timestamp_ != timestamp) {
1327 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001328 last_width_ = encoded_image._encodedWidth;
1329 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001330 } else {
1331 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001332 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1333 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001334 }
1335 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001336 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001337 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001338 if (num_received_layers_ == num_expected_layers_) {
1339 encoded_frame_event_.Set();
1340 }
sprangb1ca0732017-02-01 08:38:12 -08001341 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001342 }
1343
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001344 void OnEncoderConfigurationChanged(
1345 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001346 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001347 VideoEncoderConfig::ContentType content_type,
1348 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001349 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001350 ++number_of_reconfigurations_;
1351 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1352 }
1353
Per Kjellanderdcef6412020-10-07 15:09:05 +02001354 void OnBitrateAllocationUpdated(
1355 const VideoBitrateAllocation& allocation) override {
1356 MutexLock lock(&mutex_);
1357 ++number_of_bitrate_allocations_;
1358 last_bitrate_allocation_ = allocation;
1359 }
1360
Per Kjellandera9434842020-10-15 17:53:22 +02001361 void OnVideoLayersAllocationUpdated(
1362 VideoLayersAllocation allocation) override {
1363 MutexLock lock(&mutex_);
1364 ++number_of_layers_allocations_;
1365 last_layers_allocation_ = allocation;
1366 rtc::StringBuilder log;
1367 for (const auto& layer : allocation.active_spatial_layers) {
1368 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1369 << "[";
1370 for (const auto target_bitrate :
1371 layer.target_bitrate_per_temporal_layer) {
1372 log << target_bitrate.kbps() << ",";
1373 }
1374 log << "]";
1375 }
1376 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1377 }
1378
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001379 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001380 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001381 TestEncoder* test_encoder_;
1382 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001383 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001384 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001385 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001386 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001387 uint32_t last_height_ = 0;
1388 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001389 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001390 size_t num_expected_layers_ = 1;
1391 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001392 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001393 int number_of_reconfigurations_ = 0;
1394 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001395 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1396 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001397 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1398 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001399 };
1400
Sergey Silkin5ee69672019-07-02 14:18:34 +02001401 class VideoBitrateAllocatorProxyFactory
1402 : public VideoBitrateAllocatorFactory {
1403 public:
1404 VideoBitrateAllocatorProxyFactory()
1405 : bitrate_allocator_factory_(
1406 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1407
1408 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1409 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001410 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001411 codec_config_ = codec;
1412 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1413 }
1414
1415 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001416 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001417 return codec_config_;
1418 }
1419
1420 private:
1421 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1422
Markus Handella3765182020-07-08 13:13:32 +02001423 mutable Mutex mutex_;
1424 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001425 };
1426
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001427 Clock* clock() { return time_controller_.GetClock(); }
1428 void AdvanceTime(TimeDelta duration) {
1429 time_controller_.AdvanceTime(duration);
1430 }
1431
1432 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1433
1434 protected:
1435 virtual TaskQueueFactory* GetTaskQueueFactory() {
1436 return time_controller_.GetTaskQueueFactory();
1437 }
1438
1439 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001440 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001441 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001442 int codec_width_;
1443 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001444 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001445 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001446 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001447 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001448 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001449 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001450 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001451 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001452};
1453
mflodmancc3d4422017-08-03 08:27:51 -07001454TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001456 DataRate::BitsPerSec(kTargetBitrateBps),
1457 DataRate::BitsPerSec(kTargetBitrateBps),
1458 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001459 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001460 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001462 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001463 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001464}
1465
mflodmancc3d4422017-08-03 08:27:51 -07001466TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001467 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001468 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001469 // The encoder will cache up to one frame for a short duration. Adding two
1470 // frames means that the first frame will be dropped and the second frame will
1471 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001472 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001473 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001474 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001475 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001476
Henrik Boström381d1092020-05-12 18:49:07 +02001477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps),
1480 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001481
Sebastian Janssona3177052018-04-10 13:05:49 +02001482 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001483 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001484 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1485
1486 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001487 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001488}
1489
mflodmancc3d4422017-08-03 08:27:51 -07001490TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001492 DataRate::BitsPerSec(kTargetBitrateBps),
1493 DataRate::BitsPerSec(kTargetBitrateBps),
1494 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001495 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001496 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001497
Henrik Boström381d1092020-05-12 18:49:07 +02001498 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1499 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1500 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001501 // The encoder will cache up to one frame for a short duration. Adding two
1502 // frames means that the first frame will be dropped and the second frame will
1503 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001504 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001505 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001506
Henrik Boström381d1092020-05-12 18:49:07 +02001507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps),
1510 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001511 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001512 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1513 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001514 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001515}
1516
mflodmancc3d4422017-08-03 08:27:51 -07001517TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001518 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001519 DataRate::BitsPerSec(kTargetBitrateBps),
1520 DataRate::BitsPerSec(kTargetBitrateBps),
1521 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001522 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001523 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001524
1525 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001526 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001527
perkja49cbd32016-09-16 07:53:41 -07001528 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001529 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001530 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001531}
1532
mflodmancc3d4422017-08-03 08:27:51 -07001533TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps),
1537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001538
perkja49cbd32016-09-16 07:53:41 -07001539 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001540 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001541
mflodmancc3d4422017-08-03 08:27:51 -07001542 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001543 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001544 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001545 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1546 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001547}
1548
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001549class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1550 public:
1551 VideoStreamEncoderBlockedTest() {}
1552
1553 TaskQueueFactory* GetTaskQueueFactory() override {
1554 return task_queue_factory_.get();
1555 }
1556
1557 private:
1558 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1559 CreateDefaultTaskQueueFactory();
1560};
1561
1562TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001564 DataRate::BitsPerSec(kTargetBitrateBps),
1565 DataRate::BitsPerSec(kTargetBitrateBps),
1566 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001567
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001568 int dropped_count = 0;
1569 stats_proxy_->SetDroppedFrameCallback(
1570 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1571 ++dropped_count;
1572 });
1573
perkj26091b12016-09-01 01:17:40 -07001574 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001575 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001576 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001577 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1578 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001579 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1580 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001581 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001582 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001583
mflodmancc3d4422017-08-03 08:27:51 -07001584 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001585
1586 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001587}
1588
Henrik Boström56db9ff2021-03-24 09:06:45 +01001589TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001591 DataRate::BitsPerSec(kTargetBitrateBps),
1592 DataRate::BitsPerSec(kTargetBitrateBps),
1593 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001594
1595 rtc::Event frame_destroyed_event;
1596 video_source_.IncomingCapturedFrame(
1597 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001598 WaitForEncodedFrame(1);
1599 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1600 fake_encoder_.GetLastInputPixelFormat());
1601 EXPECT_EQ(fake_encoder_.codec_config().width,
1602 fake_encoder_.GetLastInputWidth());
1603 EXPECT_EQ(fake_encoder_.codec_config().height,
1604 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001605 video_stream_encoder_->Stop();
1606}
1607
Henrik Boström56db9ff2021-03-24 09:06:45 +01001608TEST_F(VideoStreamEncoderTest,
1609 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001610 // Use the cropping factory.
1611 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001612 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001613 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1614 kMaxPayloadLength);
1615 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1616
1617 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001619 DataRate::BitsPerSec(kTargetBitrateBps),
1620 DataRate::BitsPerSec(kTargetBitrateBps),
1621 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001622 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1623 WaitForEncodedFrame(1);
1624 // The encoder will have been configured once.
1625 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1626 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1627 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1628
1629 // Now send in a fake frame that needs to be cropped as the width/height
1630 // aren't divisible by 4 (see CreateEncoderStreams above).
1631 rtc::Event frame_destroyed_event;
1632 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1633 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001634 WaitForEncodedFrame(2);
1635 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1636 fake_encoder_.GetLastInputPixelFormat());
1637 EXPECT_EQ(fake_encoder_.codec_config().width,
1638 fake_encoder_.GetLastInputWidth());
1639 EXPECT_EQ(fake_encoder_.codec_config().height,
1640 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001641 video_stream_encoder_->Stop();
1642}
1643
Evan Shrubsole895556e2020-10-05 09:15:13 +02001644TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1646 DataRate::BitsPerSec(kTargetBitrateBps),
1647 DataRate::BitsPerSec(kTargetBitrateBps),
1648 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1649
1650 video_source_.IncomingCapturedFrame(
1651 CreateNV12Frame(1, codec_width_, codec_height_));
1652 WaitForEncodedFrame(1);
1653 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1654 fake_encoder_.GetLastInputPixelFormat());
1655 video_stream_encoder_->Stop();
1656}
1657
Henrik Boström56db9ff2021-03-24 09:06:45 +01001658TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1660 DataRate::BitsPerSec(kTargetBitrateBps),
1661 DataRate::BitsPerSec(kTargetBitrateBps),
1662 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1663
1664 fake_encoder_.SetPreferredPixelFormats({});
1665
1666 rtc::Event frame_destroyed_event;
1667 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1668 1, &frame_destroyed_event, codec_width_, codec_height_));
1669 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001670 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001671 fake_encoder_.GetLastInputPixelFormat());
1672 video_stream_encoder_->Stop();
1673}
1674
Henrik Boström56db9ff2021-03-24 09:06:45 +01001675TEST_F(VideoStreamEncoderTest,
1676 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps),
1680 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1681
1682 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1683
1684 rtc::Event frame_destroyed_event;
1685 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1686 1, &frame_destroyed_event, codec_width_, codec_height_));
1687 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001688 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001689 fake_encoder_.GetLastInputPixelFormat());
1690 video_stream_encoder_->Stop();
1691}
1692
Henrik Boström56db9ff2021-03-24 09:06:45 +01001693TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1695 DataRate::BitsPerSec(kTargetBitrateBps),
1696 DataRate::BitsPerSec(kTargetBitrateBps),
1697 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1698
1699 // Fake NV12 native frame does not allow mapping to I444.
1700 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1701
1702 rtc::Event frame_destroyed_event;
1703 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1704 1, &frame_destroyed_event, codec_width_, codec_height_));
1705 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001706 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001707 fake_encoder_.GetLastInputPixelFormat());
1708 video_stream_encoder_->Stop();
1709}
1710
Henrik Boström56db9ff2021-03-24 09:06:45 +01001711TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1713 DataRate::BitsPerSec(kTargetBitrateBps),
1714 DataRate::BitsPerSec(kTargetBitrateBps),
1715 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1716
1717 rtc::Event frame_destroyed_event;
1718 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1719 1, &frame_destroyed_event, codec_width_, codec_height_));
1720 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001721 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001722 fake_encoder_.GetLastInputPixelFormat());
1723 video_stream_encoder_->Stop();
1724}
1725
Ying Wang9b881ab2020-02-07 14:29:32 +01001726TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001728 DataRate::BitsPerSec(kTargetBitrateBps),
1729 DataRate::BitsPerSec(kTargetBitrateBps),
1730 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001731 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1732 WaitForEncodedFrame(1);
1733
Henrik Boström381d1092020-05-12 18:49:07 +02001734 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001735 DataRate::BitsPerSec(kTargetBitrateBps),
1736 DataRate::BitsPerSec(kTargetBitrateBps),
1737 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001738 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1739 // frames. Adding two frames means that the first frame will be dropped and
1740 // the second frame will be sent to the encoder.
1741 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1742 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1743 WaitForEncodedFrame(3);
1744 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1745 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1746 WaitForEncodedFrame(5);
1747 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1748 video_stream_encoder_->Stop();
1749}
1750
mflodmancc3d4422017-08-03 08:27:51 -07001751TEST_F(VideoStreamEncoderTest,
1752 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001754 DataRate::BitsPerSec(kTargetBitrateBps),
1755 DataRate::BitsPerSec(kTargetBitrateBps),
1756 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001757 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001758
1759 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001760 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001761 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001762 // The encoder will have been configured once when the first frame is
1763 // received.
1764 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001765
1766 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001767 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001768 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001769 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001770 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001771
1772 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001773 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001774 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001775 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001776 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001777
mflodmancc3d4422017-08-03 08:27:51 -07001778 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001779}
1780
mflodmancc3d4422017-08-03 08:27:51 -07001781TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001783 DataRate::BitsPerSec(kTargetBitrateBps),
1784 DataRate::BitsPerSec(kTargetBitrateBps),
1785 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001786
1787 // Capture a frame and wait for it to synchronize with the encoder thread.
1788 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001789 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001790 // The encoder will have been configured once.
1791 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001792 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1793 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1794
1795 codec_width_ *= 2;
1796 codec_height_ *= 2;
1797 // Capture a frame with a higher resolution and wait for it to synchronize
1798 // with the encoder thread.
1799 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001800 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001801 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1802 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001803 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001804
mflodmancc3d4422017-08-03 08:27:51 -07001805 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001806}
1807
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001808TEST_F(VideoStreamEncoderTest,
1809 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001811 DataRate::BitsPerSec(kTargetBitrateBps),
1812 DataRate::BitsPerSec(kTargetBitrateBps),
1813 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001814
1815 // Capture a frame and wait for it to synchronize with the encoder thread.
1816 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1817 WaitForEncodedFrame(1);
1818
1819 VideoEncoderConfig video_encoder_config;
1820 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1821 // Changing the max payload data length recreates encoder.
1822 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1823 kMaxPayloadLength / 2);
1824
1825 // Capture a frame and wait for it to synchronize with the encoder thread.
1826 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1827 WaitForEncodedFrame(2);
1828 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1829
1830 video_stream_encoder_->Stop();
1831}
1832
Sergey Silkin5ee69672019-07-02 14:18:34 +02001833TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001835 DataRate::BitsPerSec(kTargetBitrateBps),
1836 DataRate::BitsPerSec(kTargetBitrateBps),
1837 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001838
1839 VideoEncoderConfig video_encoder_config;
1840 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1841 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1842 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1843 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1844 kMaxPayloadLength);
1845
1846 // Capture a frame and wait for it to synchronize with the encoder thread.
1847 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1848 WaitForEncodedFrame(1);
1849 // The encoder will have been configured once when the first frame is
1850 // received.
1851 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1852 EXPECT_EQ(kTargetBitrateBps,
1853 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1854 EXPECT_EQ(kStartBitrateBps,
1855 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1856
Sergey Silkin6456e352019-07-08 17:56:40 +02001857 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1858 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001859 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1860 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1861 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1862 kMaxPayloadLength);
1863
1864 // Capture a frame and wait for it to synchronize with the encoder thread.
1865 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1866 WaitForEncodedFrame(2);
1867 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1868 // Bitrate limits have changed - rate allocator should be reconfigured,
1869 // encoder should not be reconfigured.
1870 EXPECT_EQ(kTargetBitrateBps * 2,
1871 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1872 EXPECT_EQ(kStartBitrateBps * 2,
1873 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1874 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1875
1876 video_stream_encoder_->Stop();
1877}
1878
Sergey Silkin6456e352019-07-08 17:56:40 +02001879TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001880 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001882 DataRate::BitsPerSec(kTargetBitrateBps),
1883 DataRate::BitsPerSec(kTargetBitrateBps),
1884 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001885
Sergey Silkincd02eba2020-01-20 14:48:40 +01001886 const uint32_t kMinEncBitrateKbps = 100;
1887 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001888 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001889 /*frame_size_pixels=*/codec_width_ * codec_height_,
1890 /*min_start_bitrate_bps=*/0,
1891 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1892 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001893 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1894
Sergey Silkincd02eba2020-01-20 14:48:40 +01001895 VideoEncoderConfig video_encoder_config;
1896 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1897 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1898 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1899 (kMinEncBitrateKbps + 1) * 1000;
1900 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1901 kMaxPayloadLength);
1902
1903 // When both encoder and app provide bitrate limits, the intersection of
1904 // provided sets should be used.
1905 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1906 WaitForEncodedFrame(1);
1907 EXPECT_EQ(kMaxEncBitrateKbps,
1908 bitrate_allocator_factory_.codec_config().maxBitrate);
1909 EXPECT_EQ(kMinEncBitrateKbps + 1,
1910 bitrate_allocator_factory_.codec_config().minBitrate);
1911
1912 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1913 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1914 (kMinEncBitrateKbps - 1) * 1000;
1915 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1916 kMaxPayloadLength);
1917 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001918 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001919 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001920 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001921 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001922 bitrate_allocator_factory_.codec_config().minBitrate);
1923
Sergey Silkincd02eba2020-01-20 14:48:40 +01001924 video_stream_encoder_->Stop();
1925}
1926
1927TEST_F(VideoStreamEncoderTest,
1928 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001930 DataRate::BitsPerSec(kTargetBitrateBps),
1931 DataRate::BitsPerSec(kTargetBitrateBps),
1932 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001933
1934 const uint32_t kMinAppBitrateKbps = 100;
1935 const uint32_t kMaxAppBitrateKbps = 200;
1936 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1937 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1938 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1939 /*frame_size_pixels=*/codec_width_ * codec_height_,
1940 /*min_start_bitrate_bps=*/0,
1941 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1942 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1943 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1944
1945 VideoEncoderConfig video_encoder_config;
1946 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1947 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1948 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1949 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001950 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1951 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001952
Sergey Silkincd02eba2020-01-20 14:48:40 +01001953 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1954 WaitForEncodedFrame(1);
1955 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001956 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001957 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001958 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001959
1960 video_stream_encoder_->Stop();
1961}
1962
1963TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001965 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001966 DataRate::BitsPerSec(kTargetBitrateBps),
1967 DataRate::BitsPerSec(kTargetBitrateBps),
1968 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001969
1970 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001971 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001972 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001973 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001974 fake_encoder_.SetResolutionBitrateLimits(
1975 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1976
1977 VideoEncoderConfig video_encoder_config;
1978 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1979 video_encoder_config.max_bitrate_bps = 0;
1980 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1981 kMaxPayloadLength);
1982
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001983 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001984 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1985 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001986 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1987 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001988 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1989 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1990
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001992 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1993 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001994 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1995 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001996 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1997 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1998
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001999 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002000 // encoder for 360p should be used.
2001 video_source_.IncomingCapturedFrame(
2002 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2003 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002006 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2007 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2008
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002009 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002010 // ignored.
2011 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2012 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002013 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2014 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002015 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2016 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2018 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002019 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2020 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2021
2022 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2023 // for 270p should be used.
2024 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2025 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002026 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2027 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002028 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2029 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2030
2031 video_stream_encoder_->Stop();
2032}
2033
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002034TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002035 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002036 DataRate::BitsPerSec(kTargetBitrateBps),
2037 DataRate::BitsPerSec(kTargetBitrateBps),
2038 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002039
2040 VideoEncoderConfig video_encoder_config;
2041 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2042 video_encoder_config.max_bitrate_bps = 0;
2043 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2044 kMaxPayloadLength);
2045
2046 // Encode 720p frame to get the default encoder target bitrate.
2047 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2048 WaitForEncodedFrame(1);
2049 const uint32_t kDefaultTargetBitrateFor720pKbps =
2050 bitrate_allocator_factory_.codec_config()
2051 .simulcastStream[0]
2052 .targetBitrate;
2053
2054 // Set the max recommended encoder bitrate to something lower than the default
2055 // target bitrate.
2056 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2057 1280 * 720, 10 * 1000, 10 * 1000,
2058 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2059 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2060
2061 // Change resolution to trigger encoder reinitialization.
2062 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2063 WaitForEncodedFrame(2);
2064 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2065 WaitForEncodedFrame(3);
2066
2067 // Ensure the target bitrate is capped by the max bitrate.
2068 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2069 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2070 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2071 .simulcastStream[0]
2072 .targetBitrate *
2073 1000,
2074 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2075
2076 video_stream_encoder_->Stop();
2077}
2078
Åsa Perssona7e34d32021-01-20 15:36:13 +01002079TEST_F(VideoStreamEncoderTest,
2080 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2081 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2082 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2083 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2084 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2085 fake_encoder_.SetResolutionBitrateLimits(
2086 {kEncoderLimits270p, kEncoderLimits360p});
2087
2088 // Two streams, highest stream active.
2089 VideoEncoderConfig config;
2090 const int kNumStreams = 2;
2091 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2092 config.max_bitrate_bps = 0;
2093 config.simulcast_layers[0].active = false;
2094 config.simulcast_layers[1].active = true;
2095 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002096 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002097 "VP8", /*max qp*/ 56, /*screencast*/ false,
2098 /*screenshare enabled*/ false);
2099 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2100
2101 // The encoder bitrate limits for 270p should be used.
2102 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2103 EXPECT_FALSE(WaitForFrame(1000));
2104 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2105 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2106 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2107 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2108 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2109
2110 // The encoder bitrate limits for 360p should be used.
2111 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2112 EXPECT_FALSE(WaitForFrame(1000));
2113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2114 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2115 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2116 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2117
2118 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2119 video_source_.IncomingCapturedFrame(
2120 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2121 EXPECT_FALSE(WaitForFrame(1000));
2122 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2123 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2124 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2125 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2126
2127 // Resolution higher than 360p. Encoder limits should be ignored.
2128 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2129 EXPECT_FALSE(WaitForFrame(1000));
2130 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2131 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2132 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2133 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2134 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2135 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2136 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2137 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2138
2139 // Resolution lower than 270p. The encoder limits for 270p should be used.
2140 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2141 EXPECT_FALSE(WaitForFrame(1000));
2142 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2143 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2144 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2145 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2146
2147 video_stream_encoder_->Stop();
2148}
2149
2150TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002151 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2152 // Two streams, highest stream active.
2153 VideoEncoderConfig config;
2154 const int kNumStreams = 2;
2155 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2156 config.max_bitrate_bps = 0;
2157 config.simulcast_layers[0].active = false;
2158 config.simulcast_layers[1].active = true;
2159 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002160 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002161 "VP8", /*max qp*/ 56, /*screencast*/ false,
2162 /*screenshare enabled*/ false);
2163 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2164
2165 // Default bitrate limits for 270p should be used.
2166 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2167 kDefaultLimits270p =
2168 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002169 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002170 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2171 EXPECT_FALSE(WaitForFrame(1000));
2172 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2173 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2174 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2175 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2176 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2177
2178 // Default bitrate limits for 360p should be used.
2179 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2180 kDefaultLimits360p =
2181 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002182 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002183 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2184 EXPECT_FALSE(WaitForFrame(1000));
2185 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2186 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2187 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2188 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2189
2190 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2191 video_source_.IncomingCapturedFrame(
2192 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2193 EXPECT_FALSE(WaitForFrame(1000));
2194 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2195 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2196 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2197 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2198
2199 // Default bitrate limits for 540p should be used.
2200 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2201 kDefaultLimits540p =
2202 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002203 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002204 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2205 EXPECT_FALSE(WaitForFrame(1000));
2206 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2207 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2208 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2209 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2210
2211 video_stream_encoder_->Stop();
2212}
2213
2214TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2216 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2217 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2218 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2219 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2220 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2221 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2222 fake_encoder_.SetResolutionBitrateLimits(
2223 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2224
2225 // Three streams, middle stream active.
2226 VideoEncoderConfig config;
2227 const int kNumStreams = 3;
2228 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2229 config.simulcast_layers[0].active = false;
2230 config.simulcast_layers[1].active = true;
2231 config.simulcast_layers[2].active = false;
2232 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002233 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002234 "VP8", /*max qp*/ 56, /*screencast*/ false,
2235 /*screenshare enabled*/ false);
2236 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2237
2238 // The encoder bitrate limits for 360p should be used.
2239 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2240 EXPECT_FALSE(WaitForFrame(1000));
2241 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2242 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2243 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2244 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2245 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2246
2247 // The encoder bitrate limits for 270p should be used.
2248 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2249 EXPECT_FALSE(WaitForFrame(1000));
2250 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2251 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2252 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2253 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2254
2255 video_stream_encoder_->Stop();
2256}
2257
2258TEST_F(VideoStreamEncoderTest,
2259 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2260 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2261 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2262 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2263 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2264 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2265 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2266 fake_encoder_.SetResolutionBitrateLimits(
2267 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2268
2269 // Three streams, lowest stream active.
2270 VideoEncoderConfig config;
2271 const int kNumStreams = 3;
2272 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2273 config.simulcast_layers[0].active = true;
2274 config.simulcast_layers[1].active = false;
2275 config.simulcast_layers[2].active = false;
2276 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002277 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002278 "VP8", /*max qp*/ 56, /*screencast*/ false,
2279 /*screenshare enabled*/ false);
2280 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2281
2282 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2283 // on lowest stream, limits for 270p should not be used
2284 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2285 EXPECT_FALSE(WaitForFrame(1000));
2286 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2287 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2288 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2289 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2290 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2291
2292 video_stream_encoder_->Stop();
2293}
2294
2295TEST_F(VideoStreamEncoderTest,
2296 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2297 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2298 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2299 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2300 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2301 fake_encoder_.SetResolutionBitrateLimits(
2302 {kEncoderLimits270p, kEncoderLimits360p});
2303 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2304
2305 // Two streams, highest stream active.
2306 VideoEncoderConfig config;
2307 const int kNumStreams = 2;
2308 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2309 config.simulcast_layers[0].active = false;
2310 config.simulcast_layers[1].active = true;
2311 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2312 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002313 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002314 "VP8", /*max qp*/ 56, /*screencast*/ false,
2315 /*screenshare enabled*/ false);
2316 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2317
2318 // The encoder bitrate limits for 270p should be used.
2319 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2320 EXPECT_FALSE(WaitForFrame(1000));
2321 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2322 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2323 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2324 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2325 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2326
2327 // The max configured bitrate is less than the encoder limit for 360p.
2328 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2329 EXPECT_FALSE(WaitForFrame(1000));
2330 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2331 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2332 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2333 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2334
2335 video_stream_encoder_->Stop();
2336}
2337
mflodmancc3d4422017-08-03 08:27:51 -07002338TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002339 EXPECT_TRUE(video_source_.has_sinks());
2340 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002341 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002342 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002343 EXPECT_FALSE(video_source_.has_sinks());
2344 EXPECT_TRUE(new_video_source.has_sinks());
2345
mflodmancc3d4422017-08-03 08:27:51 -07002346 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002347}
2348
mflodmancc3d4422017-08-03 08:27:51 -07002349TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002350 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002351 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002352 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002353 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002354}
2355
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002356class ResolutionAlignmentTest
2357 : public VideoStreamEncoderTest,
2358 public ::testing::WithParamInterface<
2359 ::testing::tuple<int, std::vector<double>>> {
2360 public:
2361 ResolutionAlignmentTest()
2362 : requested_alignment_(::testing::get<0>(GetParam())),
2363 scale_factors_(::testing::get<1>(GetParam())) {}
2364
2365 protected:
2366 const int requested_alignment_;
2367 const std::vector<double> scale_factors_;
2368};
2369
2370INSTANTIATE_TEST_SUITE_P(
2371 AlignmentAndScaleFactors,
2372 ResolutionAlignmentTest,
2373 ::testing::Combine(
2374 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2375 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2376 std::vector<double>{-1.0, -1.0},
2377 std::vector<double>{-1.0, -1.0, -1.0},
2378 std::vector<double>{4.0, 2.0, 1.0},
2379 std::vector<double>{9999.0, -1.0, 1.0},
2380 std::vector<double>{3.99, 2.01, 1.0},
2381 std::vector<double>{4.9, 1.7, 1.25},
2382 std::vector<double>{10.0, 4.0, 3.0},
2383 std::vector<double>{1.75, 3.5},
2384 std::vector<double>{1.5, 2.5},
2385 std::vector<double>{1.3, 1.0})));
2386
2387TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2388 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002389 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002390 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2391 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2392
2393 // Fill config with the scaling factor by which to reduce encoding size.
2394 const int num_streams = scale_factors_.size();
2395 VideoEncoderConfig config;
2396 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2397 for (int i = 0; i < num_streams; ++i) {
2398 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2399 }
2400 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002401 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002402 "VP8", /*max qp*/ 56, /*screencast*/ false,
2403 /*screenshare enabled*/ false);
2404 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2405
Henrik Boström381d1092020-05-12 18:49:07 +02002406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002407 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2408 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2409 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2410 // Wait for all layers before triggering event.
2411 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002412
2413 // On the 1st frame, we should have initialized the encoder and
2414 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002415 int64_t timestamp_ms = kFrameIntervalMs;
2416 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2417 WaitForEncodedFrame(timestamp_ms);
2418 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002419
2420 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2421 // (It's up the to the encoder to potentially drop the previous frame,
2422 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002423 timestamp_ms += kFrameIntervalMs;
2424 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2425 WaitForEncodedFrame(timestamp_ms);
2426 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2427
2428 VideoCodec codec = fake_encoder_.video_codec();
2429 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2430 // Frame size should be a multiple of the requested alignment.
2431 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2432 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2433 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2434 // Aspect ratio should match.
2435 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2436 codec.height * codec.simulcastStream[i].width);
2437 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002438
2439 video_stream_encoder_->Stop();
2440}
2441
Jonathan Yubc771b72017-12-08 17:04:29 -08002442TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2443 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002444 const int kWidth = 1280;
2445 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002446
2447 // We rely on the automatic resolution adaptation, but we handle framerate
2448 // adaptation manually by mocking the stats proxy.
2449 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002450
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002451 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002453 DataRate::BitsPerSec(kTargetBitrateBps),
2454 DataRate::BitsPerSec(kTargetBitrateBps),
2455 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002456 video_stream_encoder_->SetSource(&video_source_,
2457 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002458 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002459 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002460 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002461 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2462
Jonathan Yubc771b72017-12-08 17:04:29 -08002463 // Adapt down as far as possible.
2464 rtc::VideoSinkWants last_wants;
2465 int64_t t = 1;
2466 int loop_count = 0;
2467 do {
2468 ++loop_count;
2469 last_wants = video_source_.sink_wants();
2470
2471 // Simulate the framerate we've been asked to adapt to.
2472 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2473 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2474 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2475 mock_stats.input_frame_rate = fps;
2476 stats_proxy_->SetMockStats(mock_stats);
2477
2478 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2479 sink_.WaitForEncodedFrame(t);
2480 t += frame_interval_ms;
2481
mflodmancc3d4422017-08-03 08:27:51 -07002482 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002483 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002484 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002485 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2486 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002487 } while (video_source_.sink_wants().max_pixel_count <
2488 last_wants.max_pixel_count ||
2489 video_source_.sink_wants().max_framerate_fps <
2490 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002491
Jonathan Yubc771b72017-12-08 17:04:29 -08002492 // Verify that we've adapted all the way down.
2493 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002494 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002495 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2496 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002497 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002498 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2499 *video_source_.last_sent_height());
2500 EXPECT_EQ(kMinBalancedFramerateFps,
2501 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002502
Jonathan Yubc771b72017-12-08 17:04:29 -08002503 // Adapt back up the same number of times we adapted down.
2504 for (int i = 0; i < loop_count - 1; ++i) {
2505 last_wants = video_source_.sink_wants();
2506
2507 // Simulate the framerate we've been asked to adapt to.
2508 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2509 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2510 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2511 mock_stats.input_frame_rate = fps;
2512 stats_proxy_->SetMockStats(mock_stats);
2513
2514 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2515 sink_.WaitForEncodedFrame(t);
2516 t += frame_interval_ms;
2517
Henrik Boström91aa7322020-04-28 12:24:33 +02002518 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002519 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002520 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002521 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2522 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002523 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2524 last_wants.max_pixel_count ||
2525 video_source_.sink_wants().max_framerate_fps >
2526 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002527 }
2528
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002529 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002530 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002532 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2533 EXPECT_EQ((loop_count - 1) * 2,
2534 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002535
mflodmancc3d4422017-08-03 08:27:51 -07002536 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002537}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002538
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002539TEST_F(VideoStreamEncoderTest,
2540 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2541 video_stream_encoder_->OnBitrateUpdated(
2542 DataRate::BitsPerSec(kTargetBitrateBps),
2543 DataRate::BitsPerSec(kTargetBitrateBps),
2544 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002545 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002546
2547 const int kFrameWidth = 1280;
2548 const int kFrameHeight = 720;
2549
2550 int64_t ntp_time = kFrameIntervalMs;
2551
2552 // Force an input frame rate to be available, or the adaptation call won't
2553 // know what framerate to adapt form.
2554 const int kInputFps = 30;
2555 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2556 stats.input_frame_rate = kInputFps;
2557 stats_proxy_->SetMockStats(stats);
2558
2559 video_source_.set_adaptation_enabled(true);
2560 video_stream_encoder_->SetSource(
2561 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002562 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002563 video_source_.IncomingCapturedFrame(
2564 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2565 sink_.WaitForEncodedFrame(ntp_time);
2566 ntp_time += kFrameIntervalMs;
2567
2568 // Trigger CPU overuse.
2569 video_stream_encoder_->TriggerCpuOveruse();
2570 video_source_.IncomingCapturedFrame(
2571 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2572 sink_.WaitForEncodedFrame(ntp_time);
2573 ntp_time += kFrameIntervalMs;
2574
2575 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2576 EXPECT_EQ(std::numeric_limits<int>::max(),
2577 video_source_.sink_wants().max_pixel_count);
2578 // Some framerate constraint should be set.
2579 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2580 EXPECT_LT(restricted_fps, kInputFps);
2581 video_source_.IncomingCapturedFrame(
2582 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2583 sink_.WaitForEncodedFrame(ntp_time);
2584 ntp_time += 100;
2585
Henrik Boström2671dac2020-05-19 16:29:09 +02002586 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002587 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2588 // Give the encoder queue time to process the change in degradation preference
2589 // by waiting for an encoded frame.
2590 video_source_.IncomingCapturedFrame(
2591 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2592 sink_.WaitForEncodedFrame(ntp_time);
2593 ntp_time += kFrameIntervalMs;
2594
2595 video_stream_encoder_->TriggerQualityLow();
2596 video_source_.IncomingCapturedFrame(
2597 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2598 sink_.WaitForEncodedFrame(ntp_time);
2599 ntp_time += kFrameIntervalMs;
2600
2601 // Some resolution constraint should be set.
2602 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2603 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2604 kFrameWidth * kFrameHeight);
2605 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2606
2607 int pixel_count = video_source_.sink_wants().max_pixel_count;
2608 // Triggering a CPU underuse should not change the sink wants since it has
2609 // not been overused for resolution since we changed degradation preference.
2610 video_stream_encoder_->TriggerCpuUnderuse();
2611 video_source_.IncomingCapturedFrame(
2612 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2613 sink_.WaitForEncodedFrame(ntp_time);
2614 ntp_time += kFrameIntervalMs;
2615 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2616 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2617
Evan Shrubsole64469032020-06-11 10:45:29 +02002618 // Change the degradation preference back. CPU underuse should not adapt since
2619 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002620 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002621 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2622 video_source_.IncomingCapturedFrame(
2623 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2624 sink_.WaitForEncodedFrame(ntp_time);
2625 ntp_time += 100;
2626 // Resolution adaptations is gone after changing degradation preference.
2627 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2628 EXPECT_EQ(std::numeric_limits<int>::max(),
2629 video_source_.sink_wants().max_pixel_count);
2630 // The fps adaptation from above is now back.
2631 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2632
2633 // Trigger CPU underuse.
2634 video_stream_encoder_->TriggerCpuUnderuse();
2635 video_source_.IncomingCapturedFrame(
2636 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2637 sink_.WaitForEncodedFrame(ntp_time);
2638 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002639 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2640
2641 // Trigger QP underuse, fps should return to normal.
2642 video_stream_encoder_->TriggerQualityHigh();
2643 video_source_.IncomingCapturedFrame(
2644 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2645 sink_.WaitForEncodedFrame(ntp_time);
2646 ntp_time += kFrameIntervalMs;
2647 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002648
2649 video_stream_encoder_->Stop();
2650}
2651
mflodmancc3d4422017-08-03 08:27:51 -07002652TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002654 DataRate::BitsPerSec(kTargetBitrateBps),
2655 DataRate::BitsPerSec(kTargetBitrateBps),
2656 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002657 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002658
sprangc5d62e22017-04-02 23:53:04 -07002659 const int kFrameWidth = 1280;
2660 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002661
Åsa Persson8c1bf952018-09-13 10:42:19 +02002662 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002663
kthelgason5e13d412016-12-01 03:59:51 -08002664 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002665 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002666 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002667 frame_timestamp += kFrameIntervalMs;
2668
perkj803d97f2016-11-01 11:45:46 -07002669 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002670 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002671 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002672 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002673 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002674 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002675
asapersson0944a802017-04-07 00:57:58 -07002676 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002677 // wanted resolution.
2678 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2679 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2680 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002681 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002682
2683 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002684 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002685 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002686 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002687 // Give the encoder queue time to process the change in degradation preference
2688 // by waiting for an encoded frame.
2689 new_video_source.IncomingCapturedFrame(
2690 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2691 sink_.WaitForEncodedFrame(frame_timestamp);
2692 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002693 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002694 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002695
sprangc5d62e22017-04-02 23:53:04 -07002696 // Force an input frame rate to be available, or the adaptation call won't
2697 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002698 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002699 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002700 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002701 stats_proxy_->SetMockStats(stats);
2702
mflodmancc3d4422017-08-03 08:27:51 -07002703 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002704 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002705 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002706 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002707 frame_timestamp += kFrameIntervalMs;
2708
2709 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002710 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002711 EXPECT_EQ(std::numeric_limits<int>::max(),
2712 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002713 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002714
asapersson02465b82017-04-10 01:12:52 -07002715 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002716 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2717 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002718 // Give the encoder queue time to process the change in degradation preference
2719 // by waiting for an encoded frame.
2720 new_video_source.IncomingCapturedFrame(
2721 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2722 sink_.WaitForEncodedFrame(frame_timestamp);
2723 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002724 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002725
mflodmancc3d4422017-08-03 08:27:51 -07002726 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002727 new_video_source.IncomingCapturedFrame(
2728 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002729 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002730 frame_timestamp += kFrameIntervalMs;
2731
2732 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002733 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002734
2735 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002736 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002737 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002738 // Give the encoder queue time to process the change in degradation preference
2739 // by waiting for an encoded frame.
2740 new_video_source.IncomingCapturedFrame(
2741 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2742 sink_.WaitForEncodedFrame(frame_timestamp);
2743 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002744 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2745 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002746 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002747 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002748
2749 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002750 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002751 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002752 // Give the encoder queue time to process the change in degradation preference
2753 // by waiting for an encoded frame.
2754 new_video_source.IncomingCapturedFrame(
2755 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2756 sink_.WaitForEncodedFrame(frame_timestamp);
2757 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002758 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2759 EXPECT_EQ(std::numeric_limits<int>::max(),
2760 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002761 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002762
mflodmancc3d4422017-08-03 08:27:51 -07002763 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002764}
2765
mflodmancc3d4422017-08-03 08:27:51 -07002766TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002768 DataRate::BitsPerSec(kTargetBitrateBps),
2769 DataRate::BitsPerSec(kTargetBitrateBps),
2770 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002771
asaperssonfab67072017-04-04 05:51:49 -07002772 const int kWidth = 1280;
2773 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002774 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002775 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002776 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2777 EXPECT_FALSE(stats.bw_limited_resolution);
2778 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2779
2780 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002781 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002782 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002783 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002784
2785 stats = stats_proxy_->GetStats();
2786 EXPECT_TRUE(stats.bw_limited_resolution);
2787 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2788
2789 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002790 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002791 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002792 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002793
2794 stats = stats_proxy_->GetStats();
2795 EXPECT_FALSE(stats.bw_limited_resolution);
2796 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2797 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2798
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002800}
2801
mflodmancc3d4422017-08-03 08:27:51 -07002802TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002803 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002804 DataRate::BitsPerSec(kTargetBitrateBps),
2805 DataRate::BitsPerSec(kTargetBitrateBps),
2806 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002807
2808 const int kWidth = 1280;
2809 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002810 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002811 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002812 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2813 EXPECT_FALSE(stats.cpu_limited_resolution);
2814 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2815
2816 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002817 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002818 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002820
2821 stats = stats_proxy_->GetStats();
2822 EXPECT_TRUE(stats.cpu_limited_resolution);
2823 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2824
2825 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002826 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002827 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002828 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002829
2830 stats = stats_proxy_->GetStats();
2831 EXPECT_FALSE(stats.cpu_limited_resolution);
2832 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002833 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002834
mflodmancc3d4422017-08-03 08:27:51 -07002835 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002836}
2837
mflodmancc3d4422017-08-03 08:27:51 -07002838TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002840 DataRate::BitsPerSec(kTargetBitrateBps),
2841 DataRate::BitsPerSec(kTargetBitrateBps),
2842 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002843
asaperssonfab67072017-04-04 05:51:49 -07002844 const int kWidth = 1280;
2845 const int kHeight = 720;
2846 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002847 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002848 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002849 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002850 EXPECT_FALSE(stats.cpu_limited_resolution);
2851 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2852
asaperssonfab67072017-04-04 05:51:49 -07002853 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002854 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002855 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002856 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002857 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002858 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002859 EXPECT_TRUE(stats.cpu_limited_resolution);
2860 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2861
2862 // Set new source with adaptation still enabled.
2863 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002865 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002866
asaperssonfab67072017-04-04 05:51:49 -07002867 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002868 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002869 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002870 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002871 EXPECT_TRUE(stats.cpu_limited_resolution);
2872 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2873
2874 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002875 video_stream_encoder_->SetSource(&new_video_source,
2876 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002877
asaperssonfab67072017-04-04 05:51:49 -07002878 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002879 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002880 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002881 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002882 EXPECT_FALSE(stats.cpu_limited_resolution);
2883 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2884
2885 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002886 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002887 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002888
asaperssonfab67072017-04-04 05:51:49 -07002889 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002890 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002891 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002892 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002893 EXPECT_TRUE(stats.cpu_limited_resolution);
2894 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2895
asaperssonfab67072017-04-04 05:51:49 -07002896 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002897 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002898 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002899 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002900 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002901 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002902 EXPECT_FALSE(stats.cpu_limited_resolution);
2903 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002904 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002905
mflodmancc3d4422017-08-03 08:27:51 -07002906 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002907}
2908
mflodmancc3d4422017-08-03 08:27:51 -07002909TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002911 DataRate::BitsPerSec(kTargetBitrateBps),
2912 DataRate::BitsPerSec(kTargetBitrateBps),
2913 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002914
asaperssonfab67072017-04-04 05:51:49 -07002915 const int kWidth = 1280;
2916 const int kHeight = 720;
2917 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002918 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002919 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002920 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002921 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002922 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002923
2924 // Set new source with adaptation still enabled.
2925 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002926 video_stream_encoder_->SetSource(&new_video_source,
2927 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002928
asaperssonfab67072017-04-04 05:51:49 -07002929 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002930 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002931 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002932 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002933 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002934 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002935
asaperssonfab67072017-04-04 05:51:49 -07002936 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002937 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002938 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002939 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002940 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002941 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002942 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002943 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002944
asaperssonfab67072017-04-04 05:51:49 -07002945 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002946 video_stream_encoder_->SetSource(&new_video_source,
2947 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002948
asaperssonfab67072017-04-04 05:51:49 -07002949 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002950 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002951 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002952 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002953 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002954 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002955
asapersson02465b82017-04-10 01:12:52 -07002956 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002957 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002958 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002959
asaperssonfab67072017-04-04 05:51:49 -07002960 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002961 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002962 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002963 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002964 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002965 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2966 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002967
mflodmancc3d4422017-08-03 08:27:51 -07002968 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002969}
2970
mflodmancc3d4422017-08-03 08:27:51 -07002971TEST_F(VideoStreamEncoderTest,
2972 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002974 DataRate::BitsPerSec(kTargetBitrateBps),
2975 DataRate::BitsPerSec(kTargetBitrateBps),
2976 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002977
2978 const int kWidth = 1280;
2979 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002980 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002981 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002982 video_source_.IncomingCapturedFrame(
2983 CreateFrame(timestamp_ms, kWidth, kHeight));
2984 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002985 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2986 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2987 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2988
2989 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002990 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002991 timestamp_ms += kFrameIntervalMs;
2992 video_source_.IncomingCapturedFrame(
2993 CreateFrame(timestamp_ms, kWidth, kHeight));
2994 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002995 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2996 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2997 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2998
2999 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003000 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003001 timestamp_ms += kFrameIntervalMs;
3002 video_source_.IncomingCapturedFrame(
3003 CreateFrame(timestamp_ms, kWidth, kHeight));
3004 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003005 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3006 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3007 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3008
Niels Möller4db138e2018-04-19 09:04:13 +02003009 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003010 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003011
3012 VideoEncoderConfig video_encoder_config;
3013 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3014 // Make format different, to force recreation of encoder.
3015 video_encoder_config.video_format.parameters["foo"] = "foo";
3016 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003017 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003018 timestamp_ms += kFrameIntervalMs;
3019 video_source_.IncomingCapturedFrame(
3020 CreateFrame(timestamp_ms, kWidth, kHeight));
3021 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003022 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3023 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3024 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3025
mflodmancc3d4422017-08-03 08:27:51 -07003026 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003027}
3028
mflodmancc3d4422017-08-03 08:27:51 -07003029TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003030 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003032 DataRate::BitsPerSec(kTargetBitrateBps),
3033 DataRate::BitsPerSec(kTargetBitrateBps),
3034 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3035
3036 const int kWidth = 1280;
3037 const int kHeight = 720;
3038 int sequence = 1;
3039
3040 // Enable BALANCED preference, no initial limitation.
3041 test::FrameForwarder source;
3042 video_stream_encoder_->SetSource(&source,
3043 webrtc::DegradationPreference::BALANCED);
3044 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3045 WaitForEncodedFrame(sequence++);
3046 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3047 EXPECT_FALSE(stats.cpu_limited_resolution);
3048 EXPECT_FALSE(stats.cpu_limited_framerate);
3049 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3050
3051 // Trigger CPU overuse, should now adapt down.
3052 video_stream_encoder_->TriggerCpuOveruse();
3053 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3054 WaitForEncodedFrame(sequence++);
3055 stats = stats_proxy_->GetStats();
3056 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3057
3058 // Set new degradation preference should clear restrictions since we changed
3059 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003060 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003061 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3062 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3063 WaitForEncodedFrame(sequence++);
3064 stats = stats_proxy_->GetStats();
3065 EXPECT_FALSE(stats.cpu_limited_resolution);
3066 EXPECT_FALSE(stats.cpu_limited_framerate);
3067 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3068
3069 // Force an input frame rate to be available, or the adaptation call won't
3070 // know what framerate to adapt from.
3071 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3072 mock_stats.input_frame_rate = 30;
3073 stats_proxy_->SetMockStats(mock_stats);
3074 video_stream_encoder_->TriggerCpuOveruse();
3075 stats_proxy_->ResetMockStats();
3076 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3077 WaitForEncodedFrame(sequence++);
3078
3079 // We have now adapted once.
3080 stats = stats_proxy_->GetStats();
3081 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3082
3083 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003084 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3085 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003086 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3087 WaitForEncodedFrame(sequence++);
3088 stats = stats_proxy_->GetStats();
3089 EXPECT_FALSE(stats.cpu_limited_resolution);
3090 EXPECT_FALSE(stats.cpu_limited_framerate);
3091 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3092
3093 video_stream_encoder_->Stop();
3094}
3095
3096TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003097 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003099 DataRate::BitsPerSec(kTargetBitrateBps),
3100 DataRate::BitsPerSec(kTargetBitrateBps),
3101 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003102
asapersson0944a802017-04-07 00:57:58 -07003103 const int kWidth = 1280;
3104 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003105 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003106
asaperssonfab67072017-04-04 05:51:49 -07003107 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003108 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003109 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003110 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003111 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003112 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3113
asapersson02465b82017-04-10 01:12:52 -07003114 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003116 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003117 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003118 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003119 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003120 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003121 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3122
3123 // Set new source with adaptation still enabled.
3124 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003125 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003126 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003127
3128 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003129 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003130 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003131 stats = stats_proxy_->GetStats();
3132 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003133 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003134 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3135
sprangc5d62e22017-04-02 23:53:04 -07003136 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003137 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003138 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003139 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003140 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003141 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003142 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003143 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003144 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003145 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003146 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3147
sprangc5d62e22017-04-02 23:53:04 -07003148 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003149 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003150 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3151 mock_stats.input_frame_rate = 30;
3152 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003153 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003154 stats_proxy_->ResetMockStats();
3155
3156 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003157 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003158 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003159
3160 // Framerate now adapted.
3161 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003162 EXPECT_FALSE(stats.cpu_limited_resolution);
3163 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003164 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3165
3166 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003167 video_stream_encoder_->SetSource(&new_video_source,
3168 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003169 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003170 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003171 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003172
3173 stats = stats_proxy_->GetStats();
3174 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003175 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003176 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3177
3178 // Try to trigger overuse. Should not succeed.
3179 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003180 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003181 stats_proxy_->ResetMockStats();
3182
3183 stats = stats_proxy_->GetStats();
3184 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003185 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003186 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3187
3188 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003189 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003190 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003191 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003192 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003193 stats = stats_proxy_->GetStats();
3194 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003195 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003196 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003197
3198 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003199 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003200 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003201 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003202 stats = stats_proxy_->GetStats();
3203 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003204 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003205 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3206
3207 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003208 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003209 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003210 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003211 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003212 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003213 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003214 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003215 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003216 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003217 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3218
3219 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003220 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003221 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003222 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003223 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003224 stats = stats_proxy_->GetStats();
3225 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003226 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003227 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003228 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003229
mflodmancc3d4422017-08-03 08:27:51 -07003230 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003231}
3232
mflodmancc3d4422017-08-03 08:27:51 -07003233TEST_F(VideoStreamEncoderTest,
3234 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003235 const int kWidth = 1280;
3236 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003238 DataRate::BitsPerSec(kTargetBitrateBps),
3239 DataRate::BitsPerSec(kTargetBitrateBps),
3240 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003241
asaperssonfab67072017-04-04 05:51:49 -07003242 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003243 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003244
asaperssonfab67072017-04-04 05:51:49 -07003245 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003246 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003247
asaperssonfab67072017-04-04 05:51:49 -07003248 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003249 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003250
asaperssonfab67072017-04-04 05:51:49 -07003251 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003252 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003253
kthelgason876222f2016-11-29 01:44:11 -08003254 // Expect a scale down.
3255 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003256 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003257
asapersson02465b82017-04-10 01:12:52 -07003258 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003259 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003260 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003261 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003262
asaperssonfab67072017-04-04 05:51:49 -07003263 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003265 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003266 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003267
asaperssonfab67072017-04-04 05:51:49 -07003268 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003269 EXPECT_EQ(std::numeric_limits<int>::max(),
3270 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003271
asaperssonfab67072017-04-04 05:51:49 -07003272 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003273 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003274 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003275 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003276
asapersson02465b82017-04-10 01:12:52 -07003277 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003278 EXPECT_EQ(std::numeric_limits<int>::max(),
3279 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003280
mflodmancc3d4422017-08-03 08:27:51 -07003281 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003282}
3283
mflodmancc3d4422017-08-03 08:27:51 -07003284TEST_F(VideoStreamEncoderTest,
3285 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003286 const int kWidth = 1280;
3287 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003289 DataRate::BitsPerSec(kTargetBitrateBps),
3290 DataRate::BitsPerSec(kTargetBitrateBps),
3291 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003292
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003293 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003294 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003295 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003296 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003297
3298 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003299 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003300 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003301 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3302 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3303
3304 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003306 EXPECT_THAT(source.sink_wants(),
3307 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003308 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3309 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3310 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3311
3312 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003313 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003314 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3315 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3316 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3317
mflodmancc3d4422017-08-03 08:27:51 -07003318 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003319}
3320
mflodmancc3d4422017-08-03 08:27:51 -07003321TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003322 const int kWidth = 1280;
3323 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003325 DataRate::BitsPerSec(kTargetBitrateBps),
3326 DataRate::BitsPerSec(kTargetBitrateBps),
3327 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003328
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003329 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003330 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 video_stream_encoder_->SetSource(&source,
3332 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003333 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3334 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003335 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003336
3337 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003339 EXPECT_THAT(source.sink_wants(),
3340 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003341 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3342 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3343 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3344
3345 // Trigger adapt down for same input resolution, expect no change.
3346 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3347 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003349 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3350 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3351 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3352
3353 // Trigger adapt down for larger input resolution, expect no change.
3354 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3355 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003356 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003357 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3359 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3360
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003362}
3363
mflodmancc3d4422017-08-03 08:27:51 -07003364TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003365 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3366 const int kWidth = 640;
3367 const int kHeight = 360;
3368 const int64_t kFrameIntervalMs = 150;
3369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3370 DataRate::BitsPerSec(kTargetBitrateBps),
3371 DataRate::BitsPerSec(kTargetBitrateBps),
3372 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3373
3374 // Enable BALANCED preference, no initial limitation.
3375 AdaptingFrameForwarder source(&time_controller_);
3376 source.set_adaptation_enabled(true);
3377 video_stream_encoder_->SetSource(&source,
3378 webrtc::DegradationPreference::BALANCED);
3379
3380 int64_t timestamp_ms = kFrameIntervalMs;
3381 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3382 sink_.WaitForEncodedFrame(kWidth, kHeight);
3383 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3384 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3385 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3386 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3387
3388 // Trigger adapt down, expect reduced fps (640x360@15fps).
3389 video_stream_encoder_->TriggerQualityLow();
3390 timestamp_ms += kFrameIntervalMs;
3391 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3392 sink_.WaitForEncodedFrame(timestamp_ms);
3393 EXPECT_THAT(source.sink_wants(),
3394 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3395 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3396 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3397 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3398
3399 // Source requests 270p, expect reduced resolution (480x270@15fps).
3400 source.OnOutputFormatRequest(480, 270);
3401 timestamp_ms += kFrameIntervalMs;
3402 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3403 WaitForEncodedFrame(480, 270);
3404 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3405
3406 // Trigger adapt down, expect reduced fps (480x270@10fps).
3407 video_stream_encoder_->TriggerQualityLow();
3408 timestamp_ms += kFrameIntervalMs;
3409 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3410 sink_.WaitForEncodedFrame(timestamp_ms);
3411 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3414 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3415
3416 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3417 source.OnOutputFormatRequest(320, 180);
3418 timestamp_ms += kFrameIntervalMs;
3419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3420 WaitForEncodedFrame(320, 180);
3421 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Trigger adapt down, expect reduced fps (320x180@7fps).
3424 video_stream_encoder_->TriggerQualityLow();
3425 timestamp_ms += kFrameIntervalMs;
3426 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3427 sink_.WaitForEncodedFrame(timestamp_ms);
3428 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3431 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3432
3433 // Source requests VGA, expect increased resolution (640x360@7fps).
3434 source.OnOutputFormatRequest(640, 360);
3435 timestamp_ms += kFrameIntervalMs;
3436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3437 WaitForEncodedFrame(timestamp_ms);
3438 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3439
3440 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3441 video_stream_encoder_->TriggerQualityHigh();
3442 timestamp_ms += kFrameIntervalMs;
3443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3444 WaitForEncodedFrame(timestamp_ms);
3445 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3448 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3451 video_stream_encoder_->TriggerQualityHigh();
3452 timestamp_ms += kFrameIntervalMs;
3453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3454 WaitForEncodedFrame(timestamp_ms);
3455 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3456 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3457 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3458 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3459
3460 // Trigger adapt up, expect increased fps (640x360@maxfps).
3461 video_stream_encoder_->TriggerQualityHigh();
3462 timestamp_ms += kFrameIntervalMs;
3463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3464 WaitForEncodedFrame(timestamp_ms);
3465 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3468 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3469
3470 video_stream_encoder_->Stop();
3471}
3472
3473TEST_F(VideoStreamEncoderTest,
3474 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3475 const int kWidth = 1280;
3476 const int kHeight = 720;
3477 const int64_t kFrameIntervalMs = 150;
3478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
3479 DataRate::BitsPerSec(kTargetBitrateBps),
3480 DataRate::BitsPerSec(kTargetBitrateBps),
3481 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3482
3483 // Enable BALANCED preference, no initial limitation.
3484 AdaptingFrameForwarder source(&time_controller_);
3485 source.set_adaptation_enabled(true);
3486 video_stream_encoder_->SetSource(&source,
3487 webrtc::DegradationPreference::BALANCED);
3488
3489 int64_t timestamp_ms = kFrameIntervalMs;
3490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3491 sink_.WaitForEncodedFrame(kWidth, kHeight);
3492 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3494 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3495 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3498 video_stream_encoder_->TriggerQualityLow();
3499 timestamp_ms += kFrameIntervalMs;
3500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3501 sink_.WaitForEncodedFrame(timestamp_ms);
3502 EXPECT_THAT(source.sink_wants(),
3503 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3505 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3506 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3507
3508 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3509 video_stream_encoder_->TriggerQualityLow();
3510 timestamp_ms += kFrameIntervalMs;
3511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3512 sink_.WaitForEncodedFrame(timestamp_ms);
3513 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3515 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3516 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
3518 // Trigger adapt down, expect reduced fps (640x360@15fps).
3519 video_stream_encoder_->TriggerQualityLow();
3520 timestamp_ms += kFrameIntervalMs;
3521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3522 WaitForEncodedFrame(timestamp_ms);
3523 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3526 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3527
3528 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3529 source.OnOutputFormatRequest(320, 180);
3530 timestamp_ms += kFrameIntervalMs;
3531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3532 WaitForEncodedFrame(320, 180);
3533 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3534 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3535
3536 // Trigger adapt down, expect reduced fps (320x180@7fps).
3537 video_stream_encoder_->TriggerCpuOveruse();
3538 timestamp_ms += kFrameIntervalMs;
3539 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3540 WaitForEncodedFrame(timestamp_ms);
3541 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3544 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3545 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3546 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3547 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3548
3549 // Source requests HD, expect increased resolution (640x360@7fps).
3550 source.OnOutputFormatRequest(1280, 720);
3551 timestamp_ms += kFrameIntervalMs;
3552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3553 WaitForEncodedFrame(timestamp_ms);
3554 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3555 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3556
3557 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3558 video_stream_encoder_->TriggerCpuUnderuse();
3559 timestamp_ms += kFrameIntervalMs;
3560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3561 WaitForEncodedFrame(timestamp_ms);
3562 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3563 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3565 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3566 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3567 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3568 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3569
3570 // Trigger adapt up, expect increased fps (640x360@maxfps).
3571 video_stream_encoder_->TriggerQualityHigh();
3572 video_stream_encoder_->TriggerCpuUnderuse();
3573 timestamp_ms += kFrameIntervalMs;
3574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3575 WaitForEncodedFrame(timestamp_ms);
3576 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3579 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3580 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3581 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3583
3584 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3585 video_stream_encoder_->TriggerQualityHigh();
3586 video_stream_encoder_->TriggerCpuUnderuse();
3587 timestamp_ms += kFrameIntervalMs;
3588 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3589 WaitForEncodedFrame(timestamp_ms);
3590 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3592 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3593 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3594 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3595 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3596 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3597
3598 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3599 video_stream_encoder_->TriggerQualityHigh();
3600 video_stream_encoder_->TriggerCpuUnderuse();
3601 timestamp_ms += kFrameIntervalMs;
3602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3603 WaitForEncodedFrame(timestamp_ms);
3604 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3605 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3607 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3609 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3610 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3611
3612 video_stream_encoder_->Stop();
3613}
3614
3615TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003616 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003617 const int kWidth = 1280;
3618 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003620 DataRate::BitsPerSec(kTargetBitrateBps),
3621 DataRate::BitsPerSec(kTargetBitrateBps),
3622 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003623
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003624 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003625 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003626 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003627 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003628
3629 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003630 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003631 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003632 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3633 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3634
3635 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003636 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003637 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3639 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3640
mflodmancc3d4422017-08-03 08:27:51 -07003641 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003642}
3643
mflodmancc3d4422017-08-03 08:27:51 -07003644TEST_F(VideoStreamEncoderTest,
3645 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003646 const int kWidth = 1280;
3647 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003649 DataRate::BitsPerSec(kTargetBitrateBps),
3650 DataRate::BitsPerSec(kTargetBitrateBps),
3651 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003652
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003653 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003654 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003655 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003656 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003657
3658 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003659 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003660 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003661 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003662 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3663
3664 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003665 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003668 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3669
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003671}
3672
mflodmancc3d4422017-08-03 08:27:51 -07003673TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003674 const int kWidth = 1280;
3675 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003677 DataRate::BitsPerSec(kTargetBitrateBps),
3678 DataRate::BitsPerSec(kTargetBitrateBps),
3679 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003680
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003681 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003682 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003683 video_stream_encoder_->SetSource(&source,
3684 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003685
3686 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3687 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3690 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3691 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3692
3693 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003694 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003695 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003696 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3697 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3698 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3699
mflodmancc3d4422017-08-03 08:27:51 -07003700 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003701}
3702
mflodmancc3d4422017-08-03 08:27:51 -07003703TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003704 const int kWidth = 1280;
3705 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003706 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003707 DataRate::BitsPerSec(kTargetBitrateBps),
3708 DataRate::BitsPerSec(kTargetBitrateBps),
3709 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003710
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003711 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003712 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003713 video_stream_encoder_->SetSource(&source,
3714 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003715
3716 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3717 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003719 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3721 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3722
3723 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003724 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003725 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003726 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3728 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3729
mflodmancc3d4422017-08-03 08:27:51 -07003730 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003731}
3732
mflodmancc3d4422017-08-03 08:27:51 -07003733TEST_F(VideoStreamEncoderTest,
3734 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003735 const int kWidth = 1280;
3736 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003738 DataRate::BitsPerSec(kTargetBitrateBps),
3739 DataRate::BitsPerSec(kTargetBitrateBps),
3740 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003741
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003742 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003743 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003744 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003745 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003746 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003747
3748 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003749 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003750 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003751 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
3754 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003755 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003756 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003757 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003758 EXPECT_THAT(source.sink_wants(),
3759 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003760 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3761 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3762
3763 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003764 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003765 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003766 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3767 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3768 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3769
mflodmancc3d4422017-08-03 08:27:51 -07003770 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003771}
3772
mflodmancc3d4422017-08-03 08:27:51 -07003773TEST_F(VideoStreamEncoderTest,
3774 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003775 const int kWidth = 1280;
3776 const int kHeight = 720;
3777 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003779 DataRate::BitsPerSec(kTargetBitrateBps),
3780 DataRate::BitsPerSec(kTargetBitrateBps),
3781 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003782
3783 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3784 stats.input_frame_rate = kInputFps;
3785 stats_proxy_->SetMockStats(stats);
3786
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003787 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003788 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3789 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003790 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003791
3792 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003793 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003794 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3795 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003796 EXPECT_THAT(video_source_.sink_wants(),
3797 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003798
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003799 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003800 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003801 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003802 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003803 // Give the encoder queue time to process the change in degradation preference
3804 // by waiting for an encoded frame.
3805 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3806 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003807 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003808
3809 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003810 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003811 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3812 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003813 EXPECT_THAT(new_video_source.sink_wants(),
3814 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003815
3816 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003817 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003818 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003819
mflodmancc3d4422017-08-03 08:27:51 -07003820 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003821}
3822
mflodmancc3d4422017-08-03 08:27:51 -07003823TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003824 const int kWidth = 1280;
3825 const int kHeight = 720;
3826 const size_t kNumFrames = 10;
3827
Henrik Boström381d1092020-05-12 18:49:07 +02003828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003829 DataRate::BitsPerSec(kTargetBitrateBps),
3830 DataRate::BitsPerSec(kTargetBitrateBps),
3831 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003832
asaperssond0de2952017-04-21 01:47:31 -07003833 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003834 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003835 video_source_.set_adaptation_enabled(true);
3836
3837 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3838 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3839
3840 int downscales = 0;
3841 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003842 video_source_.IncomingCapturedFrame(
3843 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3844 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003845
asaperssonfab67072017-04-04 05:51:49 -07003846 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003847 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003848 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003849 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003850
3851 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3852 ++downscales;
3853
3854 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3855 EXPECT_EQ(downscales,
3856 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3857 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003858 }
mflodmancc3d4422017-08-03 08:27:51 -07003859 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003860}
3861
mflodmancc3d4422017-08-03 08:27:51 -07003862TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003863 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3864 const int kWidth = 1280;
3865 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003866 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003867 DataRate::BitsPerSec(kTargetBitrateBps),
3868 DataRate::BitsPerSec(kTargetBitrateBps),
3869 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003870
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003871 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003872 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003873 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003874 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003875 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003876
Åsa Persson8c1bf952018-09-13 10:42:19 +02003877 int64_t timestamp_ms = kFrameIntervalMs;
3878 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003879 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003880 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003881 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3882 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3883
3884 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003885 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003886 timestamp_ms += kFrameIntervalMs;
3887 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3888 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003889 EXPECT_THAT(source.sink_wants(),
3890 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003891 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3892 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3893
3894 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003895 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003898 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003900 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3901 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3902
3903 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003904 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003905 timestamp_ms += kFrameIntervalMs;
3906 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3907 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003908 EXPECT_THAT(source.sink_wants(),
3909 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003910 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3911 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3912
3913 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003914 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003917 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003919 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3920 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3921
mflodmancc3d4422017-08-03 08:27:51 -07003922 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003923}
3924
mflodmancc3d4422017-08-03 08:27:51 -07003925TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003926 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3927 const int kWidth = 1280;
3928 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003930 DataRate::BitsPerSec(kTargetBitrateBps),
3931 DataRate::BitsPerSec(kTargetBitrateBps),
3932 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003933
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003934 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003935 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003936 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003937 video_stream_encoder_->SetSource(&source,
3938 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003939
Åsa Persson8c1bf952018-09-13 10:42:19 +02003940 int64_t timestamp_ms = kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003942 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003943 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003944 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3945 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3946
3947 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003949 timestamp_ms += kFrameIntervalMs;
3950 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3951 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003952 EXPECT_THAT(source.sink_wants(),
3953 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003954 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3955 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3956
3957 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003958 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003961 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003963 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3964 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3965
3966 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003967 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003968 timestamp_ms += kFrameIntervalMs;
3969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3970 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003971 EXPECT_THAT(source.sink_wants(),
3972 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003973 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3974 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3975
3976 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003977 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003978 timestamp_ms += kFrameIntervalMs;
3979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003980 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3983 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3984
mflodmancc3d4422017-08-03 08:27:51 -07003985 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003986}
3987
Sergey Silkin41c650b2019-10-14 13:12:19 +02003988TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3989 fake_encoder_.SetResolutionBitrateLimits(
3990 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3991
Henrik Boström381d1092020-05-12 18:49:07 +02003992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003993 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3994 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3995 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3996 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003997
3998 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003999 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004000 source.set_adaptation_enabled(true);
4001 video_stream_encoder_->SetSource(
4002 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4003
4004 // Insert 720p frame.
4005 int64_t timestamp_ms = kFrameIntervalMs;
4006 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4007 WaitForEncodedFrame(1280, 720);
4008
4009 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004011 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4012 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4013 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4014 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004015 video_stream_encoder_->TriggerQualityLow();
4016
4017 // Insert 720p frame. It should be downscaled and encoded.
4018 timestamp_ms += kFrameIntervalMs;
4019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4020 WaitForEncodedFrame(960, 540);
4021
4022 // Trigger adapt up. Higher resolution should not be requested duo to lack
4023 // of bitrate.
4024 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004025 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004026
4027 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004028 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004029 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4030 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4031 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4032 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004033
4034 // Trigger adapt up. Higher resolution should be requested.
4035 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004036 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004037
4038 video_stream_encoder_->Stop();
4039}
4040
4041TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4042 fake_encoder_.SetResolutionBitrateLimits(
4043 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4044
4045 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004047 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4048 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4049 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4050 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004051
4052 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004053 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004054 source.set_adaptation_enabled(true);
4055 video_stream_encoder_->SetSource(
4056 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4057
4058 // Insert 720p frame. It should be dropped and lower resolution should be
4059 // requested.
4060 int64_t timestamp_ms = kFrameIntervalMs;
4061 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4062 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004063 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004064
4065 // Insert 720p frame. It should be downscaled and encoded.
4066 timestamp_ms += kFrameIntervalMs;
4067 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4068 WaitForEncodedFrame(960, 540);
4069
4070 video_stream_encoder_->Stop();
4071}
4072
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004073class BalancedDegradationTest : public VideoStreamEncoderTest {
4074 protected:
4075 void SetupTest() {
4076 // Reset encoder for field trials to take effect.
4077 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02004078 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004079
4080 // Enable BALANCED preference.
4081 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004082 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4083 }
4084
4085 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02004086 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004087 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
4088 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004089 }
4090
Åsa Persson45b176f2019-09-30 11:19:05 +02004091 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004092 timestamp_ms_ += kFrameIntervalMs;
4093 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004094 }
4095
4096 void InsertFrameAndWaitForEncoded() {
4097 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004098 sink_.WaitForEncodedFrame(timestamp_ms_);
4099 }
4100
4101 const int kWidth = 640; // pixels:640x360=230400
4102 const int kHeight = 360;
4103 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4104 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004105 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004106};
4107
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004108TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004109 test::ScopedFieldTrials field_trials(
4110 "WebRTC-Video-BalancedDegradationSettings/"
4111 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4112 SetupTest();
4113
4114 // Force input frame rate.
4115 const int kInputFps = 24;
4116 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4117 stats.input_frame_rate = kInputFps;
4118 stats_proxy_->SetMockStats(stats);
4119
Åsa Persson45b176f2019-09-30 11:19:05 +02004120 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004121 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004122
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004123 // Trigger adapt down, expect scaled down framerate and resolution,
4124 // since Fps diff (input-requested:0) < threshold.
4125 video_stream_encoder_->TriggerQualityLow();
4126 EXPECT_THAT(source_.sink_wants(),
4127 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004128
4129 video_stream_encoder_->Stop();
4130}
4131
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004132TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004133 test::ScopedFieldTrials field_trials(
4134 "WebRTC-Video-BalancedDegradationSettings/"
4135 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4136 SetupTest();
4137
4138 // Force input frame rate.
4139 const int kInputFps = 25;
4140 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4141 stats.input_frame_rate = kInputFps;
4142 stats_proxy_->SetMockStats(stats);
4143
Åsa Persson45b176f2019-09-30 11:19:05 +02004144 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004145 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004146
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004147 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4148 // Fps diff (input-requested:1) == threshold.
4149 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004150 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004151
4152 video_stream_encoder_->Stop();
4153}
4154
4155TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4156 test::ScopedFieldTrials field_trials(
4157 "WebRTC-Video-BalancedDegradationSettings/"
4158 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4159 SetupTest();
4160
4161 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4162
Åsa Persson45b176f2019-09-30 11:19:05 +02004163 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004164 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004165
4166 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4167 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004168 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004169
4170 video_stream_encoder_->Stop();
4171}
4172
Åsa Perssonccfb3402019-09-25 15:13:04 +02004173TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004174 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004175 "WebRTC-Video-BalancedDegradationSettings/"
4176 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004177 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004178
Åsa Persson1b247f12019-08-14 17:26:39 +02004179 const int kMinBitrateBps = 425000;
4180 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004181 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02004182
Åsa Persson45b176f2019-09-30 11:19:05 +02004183 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004184 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004185 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4186
4187 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4188 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004189 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004190 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004191 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4192
4193 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4194 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004195 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004196 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004197 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4198
Åsa Persson30ab0152019-08-27 12:22:33 +02004199 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4200 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004201 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004202 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004203 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004204 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004207 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004209 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004210
Åsa Persson30ab0152019-08-27 12:22:33 +02004211 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004212 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02004213 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004215 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004216 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4217
4218 video_stream_encoder_->Stop();
4219}
4220
Åsa Perssonccfb3402019-09-25 15:13:04 +02004221TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004222 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4223 test::ScopedFieldTrials field_trials(
4224 "WebRTC-Video-BalancedDegradationSettings/"
4225 "pixels:57600|129600|230400,fps:7|24|24/");
4226 SetupTest();
4227 OnBitrateUpdated(kLowTargetBitrateBps);
4228
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004229 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004230
4231 // Insert frame, expect scaled down:
4232 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4233 InsertFrame();
4234 EXPECT_FALSE(WaitForFrame(1000));
4235 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4236 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4237
4238 // Insert frame, expect scaled down:
4239 // resolution (320x180@24fps).
4240 InsertFrame();
4241 EXPECT_FALSE(WaitForFrame(1000));
4242 EXPECT_LT(source_.sink_wants().max_pixel_count,
4243 source_.last_wants().max_pixel_count);
4244 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4245
4246 // Frame should not be dropped (min pixels per frame reached).
4247 InsertFrameAndWaitForEncoded();
4248
4249 video_stream_encoder_->Stop();
4250}
4251
4252TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004253 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004254 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004255 "WebRTC-Video-BalancedDegradationSettings/"
4256 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004257 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004258
Åsa Persson30ab0152019-08-27 12:22:33 +02004259 const int kResolutionMinBitrateBps = 435000;
4260 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004261 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004262
Åsa Persson45b176f2019-09-30 11:19:05 +02004263 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004264 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004265 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4266
4267 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4268 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004269 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004270 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004271 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4272
4273 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4274 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004276 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004277 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4278
4279 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4280 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004281 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004282 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004283 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4284
Åsa Persson30ab0152019-08-27 12:22:33 +02004285 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4286 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004287 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004288 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004289 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4290
4291 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4292 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004293 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4295
4296 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004297 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004299 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004300 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004301 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4302
4303 video_stream_encoder_->Stop();
4304}
4305
Åsa Perssonccfb3402019-09-25 15:13:04 +02004306TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004308 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004309 "WebRTC-Video-BalancedDegradationSettings/"
4310 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004311 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004312
Åsa Persson30ab0152019-08-27 12:22:33 +02004313 const int kMinBitrateBps = 425000;
4314 const int kTooLowMinBitrateBps = 424000;
4315 const int kResolutionMinBitrateBps = 435000;
4316 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004317 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004318
Åsa Persson45b176f2019-09-30 11:19:05 +02004319 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004320 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004321 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4322
4323 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4324 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004325 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004326 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004327 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4328
4329 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4330 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004331 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004332 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004333 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4334
4335 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4336 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004337 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004338 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004339 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4340
4341 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4342 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004343 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004347 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004348 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004349 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004350 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004351 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4352
4353 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004354 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004356 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004357 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4358
4359 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004360 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004361 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004363 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
Åsa Persson1b247f12019-08-14 17:26:39 +02004366 video_stream_encoder_->Stop();
4367}
4368
mflodmancc3d4422017-08-03 08:27:51 -07004369TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004370 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4371 const int kWidth = 1280;
4372 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004374 DataRate::BitsPerSec(kTargetBitrateBps),
4375 DataRate::BitsPerSec(kTargetBitrateBps),
4376 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004377
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004378 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004379 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004380 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004381 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004382 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004383
Åsa Persson8c1bf952018-09-13 10:42:19 +02004384 int64_t timestamp_ms = kFrameIntervalMs;
4385 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004386 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004387 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004388 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4390 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4391 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4392
4393 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004394 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004395 timestamp_ms += kFrameIntervalMs;
4396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4397 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004398 EXPECT_THAT(source.sink_wants(),
4399 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004400 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4401 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4403 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4404
4405 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004406 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004407 timestamp_ms += kFrameIntervalMs;
4408 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4409 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004410 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004411 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4414 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
Jonathan Yubc771b72017-12-08 17:04:29 -08004416 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004417 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004418 timestamp_ms += kFrameIntervalMs;
4419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4420 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004421 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004422 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004424 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4426
Jonathan Yubc771b72017-12-08 17:04:29 -08004427 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004428 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004429 timestamp_ms += kFrameIntervalMs;
4430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4431 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004433 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004434 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4435 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4436 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4437 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4438
Jonathan Yubc771b72017-12-08 17:04:29 -08004439 // Trigger quality adapt down, expect no change (min resolution reached).
4440 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004441 timestamp_ms += kFrameIntervalMs;
4442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4443 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004444 EXPECT_THAT(source.sink_wants(), FpsMax());
4445 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004446 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4448 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4449 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4450
Evan Shrubsole64469032020-06-11 10:45:29 +02004451 // Trigger quality adapt up, expect upscaled resolution (480x270).
4452 video_stream_encoder_->TriggerQualityHigh();
4453 timestamp_ms += kFrameIntervalMs;
4454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4455 WaitForEncodedFrame(timestamp_ms);
4456 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
4462 // Trigger quality and cpu adapt up since both are most limited, expect
4463 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004464 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004465 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004466 timestamp_ms += kFrameIntervalMs;
4467 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4468 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004469 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4471 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4472 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004473 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004474
Evan Shrubsole64469032020-06-11 10:45:29 +02004475 // Trigger quality and cpu adapt up since both are most limited, expect
4476 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004477 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004478 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004479 timestamp_ms += kFrameIntervalMs;
4480 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4481 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004482 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004483 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004484 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004486 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4487 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004488
Evan Shrubsole64469032020-06-11 10:45:29 +02004489 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4490 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004491 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004492 timestamp_ms += kFrameIntervalMs;
4493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4494 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004495 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004496 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004498 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004499 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004500
4501 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004502 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004503 timestamp_ms += kFrameIntervalMs;
4504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004505 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004506 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004507 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004508 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4509 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004510 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004511 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004512
mflodmancc3d4422017-08-03 08:27:51 -07004513 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004514}
4515
mflodmancc3d4422017-08-03 08:27:51 -07004516TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004517 const int kWidth = 640;
4518 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004519
Henrik Boström381d1092020-05-12 18:49:07 +02004520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004521 DataRate::BitsPerSec(kTargetBitrateBps),
4522 DataRate::BitsPerSec(kTargetBitrateBps),
4523 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004524
perkj803d97f2016-11-01 11:45:46 -07004525 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004526 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004527 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004528 }
4529
mflodmancc3d4422017-08-03 08:27:51 -07004530 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004531 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004532 video_source_.IncomingCapturedFrame(CreateFrame(
4533 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004534 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004535 }
4536
mflodmancc3d4422017-08-03 08:27:51 -07004537 video_stream_encoder_->Stop();
4538 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004539 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004540
Ying Wangef3998f2019-12-09 13:06:53 +01004541 EXPECT_METRIC_EQ(
4542 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4543 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004544 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4545}
4546
mflodmancc3d4422017-08-03 08:27:51 -07004547TEST_F(VideoStreamEncoderTest,
4548 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004549 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004550 DataRate::BitsPerSec(kTargetBitrateBps),
4551 DataRate::BitsPerSec(kTargetBitrateBps),
4552 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004553 const int kWidth = 640;
4554 const int kHeight = 360;
4555
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004556 video_stream_encoder_->SetSource(&video_source_,
4557 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004558
4559 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4560 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004561 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004562 }
4563
mflodmancc3d4422017-08-03 08:27:51 -07004564 video_stream_encoder_->Stop();
4565 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004566 stats_proxy_.reset();
4567
4568 EXPECT_EQ(0,
4569 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4570}
4571
Per Kjellanderdcef6412020-10-07 15:09:05 +02004572TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4573 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004574 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004575 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004576
4577 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004578 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004579 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004580 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4581 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004582
Henrik Boström381d1092020-05-12 18:49:07 +02004583 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004584 DataRate::BitsPerSec(kLowTargetBitrateBps),
4585 DataRate::BitsPerSec(kLowTargetBitrateBps),
4586 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004587
sprang57c2fff2017-01-16 06:24:02 -08004588 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004589 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4590 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004591 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4592 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4593
Erik Språngd7329ca2019-02-21 21:19:53 +01004594 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004595 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004596 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004597
Per Kjellanderdcef6412020-10-07 15:09:05 +02004598 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004599 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004600 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4601 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004602 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004603 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004604
Per Kjellanderdcef6412020-10-07 15:09:05 +02004605 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004606 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004607 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004608 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004609 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4610 WaitForEncodedFrame(CurrentTimeMs());
4611 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004612 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004613 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004614
mflodmancc3d4422017-08-03 08:27:51 -07004615 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004616}
4617
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004618TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004619 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004620 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004621 kVideoLayersAllocation);
4622
4623 const int kDefaultFps = 30;
4624
4625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4626 DataRate::BitsPerSec(kLowTargetBitrateBps),
4627 DataRate::BitsPerSec(kLowTargetBitrateBps),
4628 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4629
4630 video_source_.IncomingCapturedFrame(
4631 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4632 WaitForEncodedFrame(CurrentTimeMs());
4633 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4634 VideoLayersAllocation last_layer_allocation =
4635 sink_.GetLastVideoLayersAllocation();
4636 // kLowTargetBitrateBps is only enough for one spatial layer.
4637 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4638
4639 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004640 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004641 // Check that encoder has been updated too, not just allocation observer.
4642 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4643 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4644
Erik Språng9d69cbe2020-10-22 17:44:42 +02004645 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004646 int number_of_layers_allocation = 1;
4647 const int64_t start_time_ms = CurrentTimeMs();
4648 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4649 video_source_.IncomingCapturedFrame(
4650 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4651 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004652 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4653 number_of_layers_allocation = sink_.number_of_layers_allocations();
4654 VideoLayersAllocation new_allocation =
4655 sink_.GetLastVideoLayersAllocation();
4656 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4657 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4658 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4659 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4660 .target_bitrate_per_temporal_layer,
4661 last_layer_allocation.active_spatial_layers[0]
4662 .target_bitrate_per_temporal_layer);
4663 last_layer_allocation = new_allocation;
4664 }
4665 }
4666 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4667 video_stream_encoder_->Stop();
4668}
4669
4670TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004671 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004672 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4673 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4674 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004675 VideoEncoderConfig video_encoder_config;
4676 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4677 /* num_streams*/ 3, &video_encoder_config);
4678 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4679 video_encoder_config.content_type =
4680 VideoEncoderConfig::ContentType::kRealtimeVideo;
4681 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004682 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004683 VideoEncoder::GetDefaultVp8Settings());
4684 for (auto& layer : video_encoder_config.simulcast_layers) {
4685 layer.num_temporal_layers = 2;
4686 }
4687 // Simulcast layers are used for enabling/disabling streams.
4688 video_encoder_config.simulcast_layers[0].active = true;
4689 video_encoder_config.simulcast_layers[1].active = false;
4690 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004691 ConfigureEncoder(std::move(video_encoder_config),
4692 VideoStreamEncoder::BitrateAllocationCallbackType::
4693 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004694
4695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4696 DataRate::BitsPerSec(kTargetBitrateBps),
4697 DataRate::BitsPerSec(kTargetBitrateBps),
4698 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4699
4700 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4701 WaitForEncodedFrame(CurrentTimeMs());
4702 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4703 VideoLayersAllocation last_layer_allocation =
4704 sink_.GetLastVideoLayersAllocation();
4705
4706 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4707 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4708 .target_bitrate_per_temporal_layer,
4709 SizeIs(2));
4710 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4711 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4712 video_stream_encoder_->Stop();
4713}
4714
4715TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004716 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004717 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004720 VideoEncoderConfig video_encoder_config;
4721 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4722 /* num_streams*/ 3, &video_encoder_config);
4723 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4724 video_encoder_config.content_type =
4725 VideoEncoderConfig::ContentType::kRealtimeVideo;
4726 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004727 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004728 VideoEncoder::GetDefaultVp8Settings());
4729 for (auto& layer : video_encoder_config.simulcast_layers) {
4730 layer.num_temporal_layers = 2;
4731 }
4732 // Simulcast layers are used for enabling/disabling streams.
4733 video_encoder_config.simulcast_layers[0].active = true;
4734 video_encoder_config.simulcast_layers[1].active = false;
4735 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004736 ConfigureEncoder(std::move(video_encoder_config),
4737 VideoStreamEncoder::BitrateAllocationCallbackType::
4738 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004739
4740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4741 DataRate::BitsPerSec(kTargetBitrateBps),
4742 DataRate::BitsPerSec(kTargetBitrateBps),
4743 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4744
4745 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4746 WaitForEncodedFrame(CurrentTimeMs());
4747 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4748 VideoLayersAllocation last_layer_allocation =
4749 sink_.GetLastVideoLayersAllocation();
4750
4751 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4752 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4753 .target_bitrate_per_temporal_layer,
4754 SizeIs(2));
4755 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4756
4757 video_stream_encoder_->Stop();
4758}
4759
4760TEST_F(VideoStreamEncoderTest,
4761 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4762 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4763 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 VideoEncoderConfig video_encoder_config;
4765 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4766 /* num_streams*/ 1, &video_encoder_config);
4767 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4768 video_encoder_config.content_type =
4769 VideoEncoderConfig::ContentType::kRealtimeVideo;
4770 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4771 vp9_settings.numberOfSpatialLayers = 2;
4772 vp9_settings.numberOfTemporalLayers = 2;
4773 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4774 vp9_settings.automaticResizeOn = false;
4775 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004776 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004777 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004778 ConfigureEncoder(std::move(video_encoder_config),
4779 VideoStreamEncoder::BitrateAllocationCallbackType::
4780 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004781
4782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4783 DataRate::BitsPerSec(kTargetBitrateBps),
4784 DataRate::BitsPerSec(kTargetBitrateBps),
4785 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4786
4787 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4788 WaitForEncodedFrame(CurrentTimeMs());
4789 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4790 VideoLayersAllocation last_layer_allocation =
4791 sink_.GetLastVideoLayersAllocation();
4792
4793 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4794 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4795 .target_bitrate_per_temporal_layer,
4796 SizeIs(2));
4797 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4798 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4799 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4800 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4801 .target_bitrate_per_temporal_layer,
4802 SizeIs(2));
4803 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4804 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4805 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4806
4807 // Since full SVC is used, expect the top layer to utilize the full target
4808 // rate.
4809 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4810 .target_bitrate_per_temporal_layer[1],
4811 DataRate::BitsPerSec(kTargetBitrateBps));
4812 video_stream_encoder_->Stop();
4813}
4814
4815TEST_F(VideoStreamEncoderTest,
4816 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4817 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4818 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004819 VideoEncoderConfig video_encoder_config;
4820 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4821 /* num_streams*/ 1, &video_encoder_config);
4822 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4823 video_encoder_config.content_type =
4824 VideoEncoderConfig::ContentType::kRealtimeVideo;
4825 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4826 vp9_settings.numberOfSpatialLayers = 2;
4827 vp9_settings.numberOfTemporalLayers = 2;
4828 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4829 vp9_settings.automaticResizeOn = false;
4830 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004831 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004832 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004833 ConfigureEncoder(std::move(video_encoder_config),
4834 VideoStreamEncoder::BitrateAllocationCallbackType::
4835 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004836
4837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4838 DataRate::BitsPerSec(kTargetBitrateBps),
4839 DataRate::BitsPerSec(kTargetBitrateBps),
4840 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4841
4842 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4843 WaitForEncodedFrame(CurrentTimeMs());
4844 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4845 VideoLayersAllocation last_layer_allocation =
4846 sink_.GetLastVideoLayersAllocation();
4847
4848 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4849 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4850 .target_bitrate_per_temporal_layer,
4851 SizeIs(1));
4852 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4853 .target_bitrate_per_temporal_layer,
4854 SizeIs(1));
4855 // Since full SVC is used, expect the top layer to utilize the full target
4856 // rate.
4857 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4858 .target_bitrate_per_temporal_layer[0],
4859 DataRate::BitsPerSec(kTargetBitrateBps));
4860 video_stream_encoder_->Stop();
4861}
4862
4863TEST_F(VideoStreamEncoderTest,
4864 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4865 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4866 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004867 VideoEncoderConfig video_encoder_config;
4868 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4869 /* num_streams*/ 1, &video_encoder_config);
4870 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4871 video_encoder_config.content_type =
4872 VideoEncoderConfig::ContentType::kRealtimeVideo;
4873 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4874 vp9_settings.numberOfSpatialLayers = 2;
4875 vp9_settings.numberOfTemporalLayers = 2;
4876 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4877 vp9_settings.automaticResizeOn = false;
4878 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004879 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004880 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004881 ConfigureEncoder(std::move(video_encoder_config),
4882 VideoStreamEncoder::BitrateAllocationCallbackType::
4883 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004884
4885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4886 DataRate::BitsPerSec(kTargetBitrateBps),
4887 DataRate::BitsPerSec(kTargetBitrateBps),
4888 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4889
4890 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4891 WaitForEncodedFrame(CurrentTimeMs());
4892 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4893 VideoLayersAllocation last_layer_allocation =
4894 sink_.GetLastVideoLayersAllocation();
4895
4896 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4897 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4898 .target_bitrate_per_temporal_layer,
4899 SizeIs(2));
4900 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4901 .target_bitrate_per_temporal_layer,
4902 SizeIs(2));
4903 // Since KSVC is, spatial layers are independend except on key frames.
4904 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4905 .target_bitrate_per_temporal_layer[1],
4906 DataRate::BitsPerSec(kTargetBitrateBps));
4907 video_stream_encoder_->Stop();
4908}
4909
4910TEST_F(VideoStreamEncoderTest,
4911 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4912 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4913 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4914 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004915 VideoEncoderConfig video_encoder_config;
4916 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4917 /* num_streams*/ 1, &video_encoder_config);
4918 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4919 video_encoder_config.content_type =
4920 VideoEncoderConfig::ContentType::kRealtimeVideo;
4921 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4922 vp9_settings.numberOfSpatialLayers = 3;
4923 vp9_settings.numberOfTemporalLayers = 2;
4924 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4925 vp9_settings.automaticResizeOn = false;
4926 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004927 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004928 vp9_settings);
4929 // Simulcast layers are used for enabling/disabling streams.
4930 video_encoder_config.simulcast_layers.resize(3);
4931 video_encoder_config.simulcast_layers[0].active = false;
4932 video_encoder_config.simulcast_layers[1].active = true;
4933 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004934 ConfigureEncoder(std::move(video_encoder_config),
4935 VideoStreamEncoder::BitrateAllocationCallbackType::
4936 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004937
4938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4939 DataRate::BitsPerSec(kTargetBitrateBps),
4940 DataRate::BitsPerSec(kTargetBitrateBps),
4941 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4942
4943 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4944 WaitForEncodedFrame(CurrentTimeMs());
4945 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4946 VideoLayersAllocation last_layer_allocation =
4947 sink_.GetLastVideoLayersAllocation();
4948
4949 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4950 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4951 .target_bitrate_per_temporal_layer,
4952 SizeIs(2));
4953 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4954 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4955
4956 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4957 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4958 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4959 .target_bitrate_per_temporal_layer,
4960 SizeIs(2));
4961 // Since full SVC is used, expect the top layer to utilize the full target
4962 // rate.
4963 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4964 .target_bitrate_per_temporal_layer[1],
4965 DataRate::BitsPerSec(kTargetBitrateBps));
4966 video_stream_encoder_->Stop();
4967}
4968
4969TEST_F(VideoStreamEncoderTest,
4970 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4971 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4972 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4973 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004974 VideoEncoderConfig video_encoder_config;
4975 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4976 /* num_streams*/ 1, &video_encoder_config);
4977 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4978 video_encoder_config.content_type =
4979 VideoEncoderConfig::ContentType::kRealtimeVideo;
4980 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4981 vp9_settings.numberOfSpatialLayers = 3;
4982 vp9_settings.numberOfTemporalLayers = 2;
4983 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4984 vp9_settings.automaticResizeOn = false;
4985 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004986 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004987 vp9_settings);
4988 // Simulcast layers are used for enabling/disabling streams.
4989 video_encoder_config.simulcast_layers.resize(3);
4990 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004991 ConfigureEncoder(std::move(video_encoder_config),
4992 VideoStreamEncoder::BitrateAllocationCallbackType::
4993 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004994
4995 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4996 DataRate::BitsPerSec(kTargetBitrateBps),
4997 DataRate::BitsPerSec(kTargetBitrateBps),
4998 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4999
5000 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5001 WaitForEncodedFrame(CurrentTimeMs());
5002 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5003 VideoLayersAllocation last_layer_allocation =
5004 sink_.GetLastVideoLayersAllocation();
5005
5006 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5007 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5008 .target_bitrate_per_temporal_layer,
5009 SizeIs(2));
5010 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5011 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5012
5013 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5014 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5015 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5016 .target_bitrate_per_temporal_layer,
5017 SizeIs(2));
5018 video_stream_encoder_->Stop();
5019}
5020
5021TEST_F(VideoStreamEncoderTest,
5022 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5023 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5024 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5025 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005026 VideoEncoderConfig video_encoder_config;
5027 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5028 /* num_streams*/ 1, &video_encoder_config);
5029 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
5030 video_encoder_config.content_type =
5031 VideoEncoderConfig::ContentType::kRealtimeVideo;
5032 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5033 vp9_settings.numberOfSpatialLayers = 3;
5034 vp9_settings.numberOfTemporalLayers = 2;
5035 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5036 vp9_settings.automaticResizeOn = false;
5037 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005038 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005039 vp9_settings);
5040 // Simulcast layers are used for enabling/disabling streams.
5041 video_encoder_config.simulcast_layers.resize(3);
5042 video_encoder_config.simulcast_layers[0].active = false;
5043 video_encoder_config.simulcast_layers[1].active = false;
5044 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005045 ConfigureEncoder(std::move(video_encoder_config),
5046 VideoStreamEncoder::BitrateAllocationCallbackType::
5047 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005048
5049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5050 DataRate::BitsPerSec(kTargetBitrateBps),
5051 DataRate::BitsPerSec(kTargetBitrateBps),
5052 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5053
5054 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5055 WaitForEncodedFrame(CurrentTimeMs());
5056 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5057 VideoLayersAllocation last_layer_allocation =
5058 sink_.GetLastVideoLayersAllocation();
5059
5060 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5061 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5062 .target_bitrate_per_temporal_layer,
5063 SizeIs(2));
5064 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5065 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5066 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5067 .target_bitrate_per_temporal_layer[1],
5068 DataRate::BitsPerSec(kTargetBitrateBps));
5069 video_stream_encoder_->Stop();
5070}
5071
5072TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5073 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005074 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005075 kVideoLayersAllocation);
5076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5077 DataRate::BitsPerSec(kTargetBitrateBps),
5078 DataRate::BitsPerSec(kTargetBitrateBps),
5079 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5080
5081 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5082 WaitForEncodedFrame(CurrentTimeMs());
5083 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5084 VideoLayersAllocation last_layer_allocation =
5085 sink_.GetLastVideoLayersAllocation();
5086
5087 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5088 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5089 .target_bitrate_per_temporal_layer,
5090 SizeIs(1));
5091 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5092 .target_bitrate_per_temporal_layer[0],
5093 DataRate::BitsPerSec(kTargetBitrateBps));
5094 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5095 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5096 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5097 video_stream_encoder_->Stop();
5098}
5099
5100TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005101 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5102 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005103 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005104 kVideoLayersAllocation);
5105
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5107 DataRate::BitsPerSec(kLowTargetBitrateBps),
5108 DataRate::BitsPerSec(kLowTargetBitrateBps),
5109 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5110
5111 video_source_.IncomingCapturedFrame(
5112 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5113 WaitForEncodedFrame(CurrentTimeMs());
5114 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5115 VideoLayersAllocation last_layer_allocation =
5116 sink_.GetLastVideoLayersAllocation();
5117 // kLowTargetBitrateBps is only enough for one spatial layer.
5118 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5119 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5120 .target_bitrate_per_temporal_layer[0],
5121 DataRate::BitsPerSec(kLowTargetBitrateBps));
5122
5123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5124 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5125 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5126 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5127 video_source_.IncomingCapturedFrame(
5128 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5129 WaitForEncodedFrame(CurrentTimeMs());
5130
5131 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5132 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5133 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5134 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5135 .target_bitrate_per_temporal_layer[0],
5136 DataRate::Zero());
5137
5138 video_stream_encoder_->Stop();
5139}
5140
Per Kjellander4190ce92020-12-15 17:24:55 +01005141TEST_F(VideoStreamEncoderTest,
5142 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5143 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005144 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005145 kVideoLayersAllocation);
5146
5147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5148 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5149 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
5150 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
5151
5152 video_source_.IncomingCapturedFrame(
5153 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5154 WaitForEncodedFrame(CurrentTimeMs());
5155 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5156 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5157 SizeIs(2));
5158 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5159 codec_width_);
5160 EXPECT_EQ(
5161 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5162 codec_height_);
5163
5164 video_source_.IncomingCapturedFrame(
5165 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5166 WaitForEncodedFrame(CurrentTimeMs());
5167 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5168 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5169 SizeIs(2));
5170 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5171 codec_width_ / 2);
5172 EXPECT_EQ(
5173 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5174 codec_height_ / 2);
5175
5176 video_stream_encoder_->Stop();
5177}
5178
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005179TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5180 // 2 TLs configured, temporal layers supported by encoder.
5181 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005182 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005183 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005184 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005185 fake_encoder_.SetTemporalLayersSupported(0, true);
5186
5187 // Bitrate allocated across temporal layers.
5188 const int kTl0Bps = kTargetBitrateBps *
5189 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005190 kNumTemporalLayers, /*temporal_id*/ 0,
5191 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005192 const int kTl1Bps = kTargetBitrateBps *
5193 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005194 kNumTemporalLayers, /*temporal_id*/ 1,
5195 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005196 VideoBitrateAllocation expected_bitrate;
5197 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5198 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5199
5200 VerifyAllocatedBitrate(expected_bitrate);
5201 video_stream_encoder_->Stop();
5202}
5203
5204TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5205 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005206 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005207 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005208 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005209 fake_encoder_.SetTemporalLayersSupported(0, false);
5210
5211 // Temporal layers not supported by the encoder.
5212 // Total bitrate should be at ti:0.
5213 VideoBitrateAllocation expected_bitrate;
5214 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
5215
5216 VerifyAllocatedBitrate(expected_bitrate);
5217 video_stream_encoder_->Stop();
5218}
5219
5220TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005221 webrtc::test::ScopedFieldTrials field_trials(
5222 "WebRTC-Video-QualityScalerSettings/"
5223 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5224 // Reset encoder for field trials to take effect.
5225 ConfigureEncoder(video_encoder_config_.Copy());
5226
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005227 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005228 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005229 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005230 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005231 fake_encoder_.SetTemporalLayersSupported(0, true);
5232 fake_encoder_.SetTemporalLayersSupported(1, false);
5233
5234 const int kS0Bps = 150000;
5235 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005236 kS0Bps *
5237 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5238 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005239 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005240 kS0Bps *
5241 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5242 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005243 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
5244 // Temporal layers not supported by si:1.
5245 VideoBitrateAllocation expected_bitrate;
5246 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5247 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5248 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5249
5250 VerifyAllocatedBitrate(expected_bitrate);
5251 video_stream_encoder_->Stop();
5252}
5253
Niels Möller7dc26b72017-12-06 10:27:48 +01005254TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5255 const int kFrameWidth = 1280;
5256 const int kFrameHeight = 720;
5257 const int kFramerate = 24;
5258
Henrik Boström381d1092020-05-12 18:49:07 +02005259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005260 DataRate::BitsPerSec(kTargetBitrateBps),
5261 DataRate::BitsPerSec(kTargetBitrateBps),
5262 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005263 test::FrameForwarder source;
5264 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005265 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005266
5267 // Insert a single frame, triggering initial configuration.
5268 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5269 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5270
5271 EXPECT_EQ(
5272 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5273 kDefaultFramerate);
5274
5275 // Trigger reconfigure encoder (without resetting the entire instance).
5276 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005277 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5278 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005279 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005280 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005281 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005282 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5283
5284 // Detector should be updated with fps limit from codec config.
5285 EXPECT_EQ(
5286 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5287 kFramerate);
5288
5289 // Trigger overuse, max framerate should be reduced.
5290 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5291 stats.input_frame_rate = kFramerate;
5292 stats_proxy_->SetMockStats(stats);
5293 video_stream_encoder_->TriggerCpuOveruse();
5294 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5295 int adapted_framerate =
5296 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5297 EXPECT_LT(adapted_framerate, kFramerate);
5298
5299 // Trigger underuse, max framerate should go back to codec configured fps.
5300 // Set extra low fps, to make sure it's actually reset, not just incremented.
5301 stats = stats_proxy_->GetStats();
5302 stats.input_frame_rate = adapted_framerate / 2;
5303 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005304 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005305 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5306 EXPECT_EQ(
5307 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5308 kFramerate);
5309
5310 video_stream_encoder_->Stop();
5311}
5312
5313TEST_F(VideoStreamEncoderTest,
5314 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5315 const int kFrameWidth = 1280;
5316 const int kFrameHeight = 720;
5317 const int kLowFramerate = 15;
5318 const int kHighFramerate = 25;
5319
Henrik Boström381d1092020-05-12 18:49:07 +02005320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005321 DataRate::BitsPerSec(kTargetBitrateBps),
5322 DataRate::BitsPerSec(kTargetBitrateBps),
5323 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005324 test::FrameForwarder source;
5325 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005326 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005327
5328 // Trigger initial configuration.
5329 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005330 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5331 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005332 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005333 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005334 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005335 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005336 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5337
5338 EXPECT_EQ(
5339 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5340 kLowFramerate);
5341
5342 // Trigger overuse, max framerate should be reduced.
5343 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5344 stats.input_frame_rate = kLowFramerate;
5345 stats_proxy_->SetMockStats(stats);
5346 video_stream_encoder_->TriggerCpuOveruse();
5347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5348 int adapted_framerate =
5349 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5350 EXPECT_LT(adapted_framerate, kLowFramerate);
5351
5352 // Reconfigure the encoder with a new (higher max framerate), max fps should
5353 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005354 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005355 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5356 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005357 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005358 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5359
5360 EXPECT_EQ(
5361 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5362 adapted_framerate);
5363
5364 // Trigger underuse, max framerate should go back to codec configured fps.
5365 stats = stats_proxy_->GetStats();
5366 stats.input_frame_rate = adapted_framerate;
5367 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005368 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005369 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5370 EXPECT_EQ(
5371 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5372 kHighFramerate);
5373
5374 video_stream_encoder_->Stop();
5375}
5376
mflodmancc3d4422017-08-03 08:27:51 -07005377TEST_F(VideoStreamEncoderTest,
5378 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005379 const int kFrameWidth = 1280;
5380 const int kFrameHeight = 720;
5381 const int kFramerate = 24;
5382
Henrik Boström381d1092020-05-12 18:49:07 +02005383 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005384 DataRate::BitsPerSec(kTargetBitrateBps),
5385 DataRate::BitsPerSec(kTargetBitrateBps),
5386 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005387 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005388 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005389 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005390
5391 // Trigger initial configuration.
5392 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005393 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5394 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005395 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005396 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005397 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005398 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005399 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005400
Niels Möller7dc26b72017-12-06 10:27:48 +01005401 EXPECT_EQ(
5402 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5403 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005404
5405 // Trigger overuse, max framerate should be reduced.
5406 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5407 stats.input_frame_rate = kFramerate;
5408 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005409 video_stream_encoder_->TriggerCpuOveruse();
5410 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005411 int adapted_framerate =
5412 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005413 EXPECT_LT(adapted_framerate, kFramerate);
5414
5415 // Change degradation preference to not enable framerate scaling. Target
5416 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005417 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005418 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005419 EXPECT_EQ(
5420 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5421 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005422
mflodmancc3d4422017-08-03 08:27:51 -07005423 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005424}
5425
mflodmancc3d4422017-08-03 08:27:51 -07005426TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005427 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005428 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005429 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5430 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5431 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005432 const int kWidth = 640;
5433 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005434
asaperssonfab67072017-04-04 05:51:49 -07005435 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005436
5437 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005438 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005439
5440 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005441 EXPECT_TRUE_WAIT(
5442 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005443
sprangc5d62e22017-04-02 23:53:04 -07005444 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005445
asaperssonfab67072017-04-04 05:51:49 -07005446 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005447 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005448 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005449
5450 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005451 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005452
Henrik Boström2671dac2020-05-19 16:29:09 +02005453 EXPECT_TRUE_WAIT(
5454 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005455
mflodmancc3d4422017-08-03 08:27:51 -07005456 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005457}
5458
mflodmancc3d4422017-08-03 08:27:51 -07005459TEST_F(VideoStreamEncoderTest,
5460 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005461 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005463 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5464 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5465 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005466 const int kWidth = 640;
5467 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005468
5469 // We expect the n initial frames to get dropped.
5470 int i;
5471 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005472 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005473 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005474 }
5475 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005476 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005477 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005478
5479 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005480 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005481
mflodmancc3d4422017-08-03 08:27:51 -07005482 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005483}
5484
mflodmancc3d4422017-08-03 08:27:51 -07005485TEST_F(VideoStreamEncoderTest,
5486 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005487 const int kWidth = 640;
5488 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005490 DataRate::BitsPerSec(kLowTargetBitrateBps),
5491 DataRate::BitsPerSec(kLowTargetBitrateBps),
5492 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005493
5494 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005495 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005496 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005497
asaperssonfab67072017-04-04 05:51:49 -07005498 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005499 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005500 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005501
mflodmancc3d4422017-08-03 08:27:51 -07005502 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005503}
5504
mflodmancc3d4422017-08-03 08:27:51 -07005505TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005506 const int kWidth = 640;
5507 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005508 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005509
5510 VideoEncoderConfig video_encoder_config;
5511 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5512 // Make format different, to force recreation of encoder.
5513 video_encoder_config.video_format.parameters["foo"] = "foo";
5514 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005515 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005517 DataRate::BitsPerSec(kLowTargetBitrateBps),
5518 DataRate::BitsPerSec(kLowTargetBitrateBps),
5519 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005520
kthelgasonb83797b2017-02-14 11:57:25 -08005521 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005522 video_stream_encoder_->SetSource(&video_source_,
5523 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005524
asaperssonfab67072017-04-04 05:51:49 -07005525 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005526 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005527 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005528
mflodmancc3d4422017-08-03 08:27:51 -07005529 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005530 fake_encoder_.SetQualityScaling(true);
5531}
5532
Åsa Persson139f4dc2019-08-02 09:29:58 +02005533TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5534 webrtc::test::ScopedFieldTrials field_trials(
5535 "WebRTC-Video-QualityScalerSettings/"
5536 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5537 // Reset encoder for field trials to take effect.
5538 ConfigureEncoder(video_encoder_config_.Copy());
5539 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5540 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5541 const int kWidth = 640;
5542 const int kHeight = 360;
5543
Henrik Boström381d1092020-05-12 18:49:07 +02005544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005545 DataRate::BitsPerSec(kTargetBitrateBps),
5546 DataRate::BitsPerSec(kTargetBitrateBps),
5547 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005548 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5549 // Frame should not be dropped.
5550 WaitForEncodedFrame(1);
5551
Henrik Boström381d1092020-05-12 18:49:07 +02005552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005553 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5554 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5555 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005556 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5557 // Frame should not be dropped.
5558 WaitForEncodedFrame(2);
5559
Henrik Boström381d1092020-05-12 18:49:07 +02005560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005561 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5562 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5563 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005564 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5565 // Expect to drop this frame, the wait should time out.
5566 ExpectDroppedFrame();
5567
5568 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005569 EXPECT_TRUE_WAIT(
5570 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005571 video_stream_encoder_->Stop();
5572}
5573
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005574TEST_F(VideoStreamEncoderTest,
5575 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5576 webrtc::test::ScopedFieldTrials field_trials(
5577 "WebRTC-Video-QualityScalerSettings/"
5578 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5579 fake_encoder_.SetQualityScaling(false);
5580 ConfigureEncoder(video_encoder_config_.Copy());
5581 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5582 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5583 const int kWidth = 640;
5584 const int kHeight = 360;
5585
5586 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5587 DataRate::BitsPerSec(kTargetBitrateBps),
5588 DataRate::BitsPerSec(kTargetBitrateBps),
5589 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5590 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5591 // Frame should not be dropped.
5592 WaitForEncodedFrame(1);
5593
5594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5595 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5596 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5597 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5598 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5599 // Frame should not be dropped.
5600 WaitForEncodedFrame(2);
5601
5602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5603 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5604 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5605 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5606 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5607 // Not dropped since quality scaling is disabled.
5608 WaitForEncodedFrame(3);
5609
5610 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005611 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005612 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5613
5614 video_stream_encoder_->Stop();
5615}
5616
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005617TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5618 const int kLowTargetBitrateBps = 400000;
5619 // Set simulcast.
5620 ResetEncoder("VP8", 3, 1, 1, false);
5621 fake_encoder_.SetQualityScaling(true);
5622 const int kWidth = 1280;
5623 const int kHeight = 720;
5624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5625 DataRate::BitsPerSec(kLowTargetBitrateBps),
5626 DataRate::BitsPerSec(kLowTargetBitrateBps),
5627 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5628 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5629 // Frame should not be dropped.
5630 WaitForEncodedFrame(1);
5631
5632 // Trigger QVGA "singlecast"
5633 // Update the config.
5634 VideoEncoderConfig video_encoder_config;
5635 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5636 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005637 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005638 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005639 "VP8", /*max qp*/ 56, /*screencast*/ false,
5640 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005641 for (auto& layer : video_encoder_config.simulcast_layers) {
5642 layer.num_temporal_layers = 1;
5643 layer.max_framerate = kDefaultFramerate;
5644 }
5645 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5646 video_encoder_config.content_type =
5647 VideoEncoderConfig::ContentType::kRealtimeVideo;
5648
5649 video_encoder_config.simulcast_layers[0].active = true;
5650 video_encoder_config.simulcast_layers[1].active = false;
5651 video_encoder_config.simulcast_layers[2].active = false;
5652
5653 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5654 kMaxPayloadLength);
5655 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5656
5657 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5658 // Frame should not be dropped.
5659 WaitForEncodedFrame(2);
5660
5661 // Trigger HD "singlecast"
5662 video_encoder_config.simulcast_layers[0].active = false;
5663 video_encoder_config.simulcast_layers[1].active = false;
5664 video_encoder_config.simulcast_layers[2].active = true;
5665
5666 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5667 kMaxPayloadLength);
5668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5669
5670 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5671 // Frame should be dropped because of initial frame drop.
5672 ExpectDroppedFrame();
5673
5674 // Expect the sink_wants to specify a scaled frame.
5675 EXPECT_TRUE_WAIT(
5676 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5677 video_stream_encoder_->Stop();
5678}
5679
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005680TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5681 const int kLowTargetBitrateBps = 400000;
5682 // Set simulcast.
5683 ResetEncoder("VP9", 1, 1, 3, false);
5684 fake_encoder_.SetQualityScaling(true);
5685 const int kWidth = 1280;
5686 const int kHeight = 720;
5687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5688 DataRate::BitsPerSec(kLowTargetBitrateBps),
5689 DataRate::BitsPerSec(kLowTargetBitrateBps),
5690 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5691 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5692 // Frame should not be dropped.
5693 WaitForEncodedFrame(1);
5694
5695 // Trigger QVGA "singlecast"
5696 // Update the config.
5697 VideoEncoderConfig video_encoder_config;
5698 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5699 &video_encoder_config);
5700 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5701 vp9_settings.numberOfSpatialLayers = 3;
5702 // Since only one layer is active - automatic resize should be enabled.
5703 vp9_settings.automaticResizeOn = true;
5704 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005705 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005706 vp9_settings);
5707 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5708 video_encoder_config.content_type =
5709 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005710 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005711 // which SVC layers are active.
5712 video_encoder_config.simulcast_layers.resize(3);
5713
5714 video_encoder_config.simulcast_layers[0].active = true;
5715 video_encoder_config.simulcast_layers[1].active = false;
5716 video_encoder_config.simulcast_layers[2].active = false;
5717
5718 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5719 kMaxPayloadLength);
5720 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5721
5722 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5723 // Frame should not be dropped.
5724 WaitForEncodedFrame(2);
5725
5726 // Trigger HD "singlecast"
5727 video_encoder_config.simulcast_layers[0].active = false;
5728 video_encoder_config.simulcast_layers[1].active = false;
5729 video_encoder_config.simulcast_layers[2].active = true;
5730
5731 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5732 kMaxPayloadLength);
5733 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5734
5735 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5736 // Frame should be dropped because of initial frame drop.
5737 ExpectDroppedFrame();
5738
5739 // Expect the sink_wants to specify a scaled frame.
5740 EXPECT_TRUE_WAIT(
5741 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5742 video_stream_encoder_->Stop();
5743}
5744
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005745TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005746 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5747 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5748 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5749 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5750 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5751 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5752 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5753 fake_encoder_.SetResolutionBitrateLimits(
5754 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5755
5756 VideoEncoderConfig video_encoder_config;
5757 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5758 &video_encoder_config);
5759 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5760 vp9_settings.numberOfSpatialLayers = 3;
5761 // Since only one layer is active - automatic resize should be enabled.
5762 vp9_settings.automaticResizeOn = true;
5763 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005764 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005765 vp9_settings);
5766 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5767 video_encoder_config.content_type =
5768 VideoEncoderConfig::ContentType::kRealtimeVideo;
5769 // Simulcast layers are used to indicate which spatial layers are active.
5770 video_encoder_config.simulcast_layers.resize(3);
5771 video_encoder_config.simulcast_layers[0].active = false;
5772 video_encoder_config.simulcast_layers[1].active = true;
5773 video_encoder_config.simulcast_layers[2].active = false;
5774
5775 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5776 kMaxPayloadLength);
5777 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5778
5779 // The encoder bitrate limits for 360p should be used.
5780 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5781 EXPECT_FALSE(WaitForFrame(1000));
5782 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5783 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5784 VideoCodecType::kVideoCodecVP9);
5785 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5786 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5787 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5788 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5789 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5790 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5791 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5792 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5793
5794 // The encoder bitrate limits for 270p should be used.
5795 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5796 EXPECT_FALSE(WaitForFrame(1000));
5797 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5798 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5799 VideoCodecType::kVideoCodecVP9);
5800 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5801 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5802 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5803 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5804 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5805 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5806 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5807 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5808
5809 video_stream_encoder_->Stop();
5810}
5811
5812TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005813 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5814 VideoEncoderConfig video_encoder_config;
5815 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5816 &video_encoder_config);
5817 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5818 vp9_settings.numberOfSpatialLayers = 3;
5819 // Since only one layer is active - automatic resize should be enabled.
5820 vp9_settings.automaticResizeOn = true;
5821 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005822 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005823 vp9_settings);
5824 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5825 video_encoder_config.content_type =
5826 VideoEncoderConfig::ContentType::kRealtimeVideo;
5827 // Simulcast layers are used to indicate which spatial layers are active.
5828 video_encoder_config.simulcast_layers.resize(3);
5829 video_encoder_config.simulcast_layers[0].active = false;
5830 video_encoder_config.simulcast_layers[1].active = true;
5831 video_encoder_config.simulcast_layers[2].active = false;
5832
5833 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5834 kMaxPayloadLength);
5835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5836
5837 // The default bitrate limits for 360p should be used.
5838 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005839 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5840 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005841 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5842 EXPECT_FALSE(WaitForFrame(1000));
5843 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5844 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5845 VideoCodecType::kVideoCodecVP9);
5846 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5847 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5848 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5849 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5850 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5851 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5852 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5853 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5854
5855 // The default bitrate limits for 270p should be used.
5856 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005857 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5858 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005859 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5860 EXPECT_FALSE(WaitForFrame(1000));
5861 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5862 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5863 VideoCodecType::kVideoCodecVP9);
5864 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5865 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5866 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5867 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5868 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5869 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5870 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5871 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5872
5873 video_stream_encoder_->Stop();
5874}
5875
5876TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5877 webrtc::test::ScopedFieldTrials field_trials(
5878 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5879 VideoEncoderConfig video_encoder_config;
5880 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5881 &video_encoder_config);
5882 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5883 vp9_settings.numberOfSpatialLayers = 3;
5884 // Since only one layer is active - automatic resize should be enabled.
5885 vp9_settings.automaticResizeOn = true;
5886 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005887 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005888 vp9_settings);
5889 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5890 video_encoder_config.content_type =
5891 VideoEncoderConfig::ContentType::kRealtimeVideo;
5892 // Simulcast layers are used to indicate which spatial layers are active.
5893 video_encoder_config.simulcast_layers.resize(3);
5894 video_encoder_config.simulcast_layers[0].active = false;
5895 video_encoder_config.simulcast_layers[1].active = true;
5896 video_encoder_config.simulcast_layers[2].active = false;
5897
5898 // Reset encoder for field trials to take effect.
5899 ConfigureEncoder(video_encoder_config.Copy());
5900
5901 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5902 kMaxPayloadLength);
5903 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5904
5905 // The default bitrate limits for 360p should not be used.
5906 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005907 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5908 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005909 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5910 EXPECT_FALSE(WaitForFrame(1000));
5911 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005912 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005913 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5914 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5915 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5916 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5917 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5918 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5919
5920 video_stream_encoder_->Stop();
5921}
5922
5923TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5924 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5925 /*num_spatial_layers=*/1, /*screenshare=*/false);
5926
5927 // The default singlecast bitrate limits for 720p should not be used.
5928 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005929 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5930 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005931 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5932 EXPECT_FALSE(WaitForFrame(1000));
5933 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5934 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5935 VideoCodecType::kVideoCodecVP9);
5936 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5937 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5938 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5939 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5940 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5941 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5942
5943 video_stream_encoder_->Stop();
5944}
5945
5946TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005947 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5948 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5949 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5950 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5951 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5952 fake_encoder_.SetResolutionBitrateLimits(
5953 {kEncoderLimits180p, kEncoderLimits720p});
5954
5955 VideoEncoderConfig video_encoder_config;
5956 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5957 &video_encoder_config);
5958 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5959 vp9_settings.numberOfSpatialLayers = 3;
5960 // Since only one layer is active - automatic resize should be enabled.
5961 vp9_settings.automaticResizeOn = true;
5962 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005963 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005964 vp9_settings);
5965 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5966 video_encoder_config.content_type =
5967 VideoEncoderConfig::ContentType::kRealtimeVideo;
5968 // Simulcast layers are used to indicate which spatial layers are active.
5969 video_encoder_config.simulcast_layers.resize(3);
5970 video_encoder_config.simulcast_layers[0].active = true;
5971 video_encoder_config.simulcast_layers[1].active = false;
5972 video_encoder_config.simulcast_layers[2].active = false;
5973
5974 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5975 kMaxPayloadLength);
5976 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5977
5978 // Limits not applied on lowest stream, limits for 180p should not be used.
5979 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5980 EXPECT_FALSE(WaitForFrame(1000));
5981 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5982 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5983 VideoCodecType::kVideoCodecVP9);
5984 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5985 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5986 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5987 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5988 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5989 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5990 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5991 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5992
5993 video_stream_encoder_->Stop();
5994}
5995
5996TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005997 InitialFrameDropActivatesWhenResolutionIncreases) {
5998 const int kWidth = 640;
5999 const int kHeight = 360;
6000
6001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6002 DataRate::BitsPerSec(kTargetBitrateBps),
6003 DataRate::BitsPerSec(kTargetBitrateBps),
6004 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6005 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6006 // Frame should not be dropped.
6007 WaitForEncodedFrame(1);
6008
6009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6010 DataRate::BitsPerSec(kLowTargetBitrateBps),
6011 DataRate::BitsPerSec(kLowTargetBitrateBps),
6012 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6013 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6014 // Frame should not be dropped, bitrate not too low for frame.
6015 WaitForEncodedFrame(2);
6016
6017 // Incoming resolution increases.
6018 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6019 // Expect to drop this frame, bitrate too low for frame.
6020 ExpectDroppedFrame();
6021
6022 // Expect the sink_wants to specify a scaled frame.
6023 EXPECT_TRUE_WAIT(
6024 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6025 video_stream_encoder_->Stop();
6026}
6027
6028TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6029 const int kWidth = 640;
6030 const int kHeight = 360;
6031 // So that quality scaling doesn't happen by itself.
6032 fake_encoder_.SetQp(kQpHigh);
6033
6034 AdaptingFrameForwarder source(&time_controller_);
6035 source.set_adaptation_enabled(true);
6036 video_stream_encoder_->SetSource(
6037 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6038
6039 int timestamp = 1;
6040
6041 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6042 DataRate::BitsPerSec(kTargetBitrateBps),
6043 DataRate::BitsPerSec(kTargetBitrateBps),
6044 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6045 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6046 WaitForEncodedFrame(timestamp);
6047 timestamp += 9000;
6048 // Long pause to disable all first BWE drop logic.
6049 AdvanceTime(TimeDelta::Millis(1000));
6050
6051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6052 DataRate::BitsPerSec(kLowTargetBitrateBps),
6053 DataRate::BitsPerSec(kLowTargetBitrateBps),
6054 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
6055 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6056 // Not dropped frame, as initial frame drop is disabled by now.
6057 WaitForEncodedFrame(timestamp);
6058 timestamp += 9000;
6059 AdvanceTime(TimeDelta::Millis(100));
6060
6061 // Quality adaptation down.
6062 video_stream_encoder_->TriggerQualityLow();
6063
6064 // Adaptation has an effect.
6065 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6066 5000);
6067
6068 // Frame isn't dropped as initial frame dropper is disabled.
6069 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6070 WaitForEncodedFrame(timestamp);
6071 timestamp += 9000;
6072 AdvanceTime(TimeDelta::Millis(100));
6073
6074 // Quality adaptation up.
6075 video_stream_encoder_->TriggerQualityHigh();
6076
6077 // Adaptation has an effect.
6078 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6079 5000);
6080
6081 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6082 // Frame should not be dropped, as initial framedropper is off.
6083 WaitForEncodedFrame(timestamp);
6084
6085 video_stream_encoder_->Stop();
6086}
6087
Åsa Persson7f354f82021-02-04 15:52:15 +01006088TEST_F(VideoStreamEncoderTest,
6089 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6090 const int kMinStartBps360p = 222000;
6091 fake_encoder_.SetResolutionBitrateLimits(
6092 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6093 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6094 800000)});
6095
6096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6097 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6098 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6099 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6100 0, 0, 0);
6101 // Frame should not be dropped, bitrate not too low for frame.
6102 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6103 WaitForEncodedFrame(1);
6104
6105 // Incoming resolution increases, initial frame drop activates.
6106 // Frame should be dropped, link allocation too low for frame.
6107 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6108 ExpectDroppedFrame();
6109
6110 // Expect sink_wants to specify a scaled frame.
6111 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6112 5000);
6113 video_stream_encoder_->Stop();
6114}
6115
6116TEST_F(VideoStreamEncoderTest,
6117 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6118 const int kMinStartBps360p = 222000;
6119 fake_encoder_.SetResolutionBitrateLimits(
6120 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6121 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6122 800000)});
6123
6124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6125 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6126 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6127 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6128 0, 0, 0);
6129 // Frame should not be dropped, bitrate not too low for frame.
6130 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6131 WaitForEncodedFrame(1);
6132
6133 // Incoming resolution increases, initial frame drop activates.
6134 // Frame should be dropped, link allocation not too low for frame.
6135 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6136 WaitForEncodedFrame(2);
6137
6138 video_stream_encoder_->Stop();
6139}
6140
Åsa Perssone644a032019-11-08 15:56:00 +01006141TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6142 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006143 "WebRTC-Video-QualityRampupSettings/"
6144 "min_pixels:921600,min_duration_ms:2000/");
6145
6146 const int kWidth = 1280;
6147 const int kHeight = 720;
6148 const int kFps = 10;
6149 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006150
6151 // Reset encoder for field trials to take effect.
6152 VideoEncoderConfig config = video_encoder_config_.Copy();
6153 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02006154 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006155 ConfigureEncoder(std::move(config));
6156 fake_encoder_.SetQp(kQpLow);
6157
6158 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006159 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006160 source.set_adaptation_enabled(true);
6161 video_stream_encoder_->SetSource(&source,
6162 DegradationPreference::MAINTAIN_FRAMERATE);
6163
6164 // Start at low bitrate.
6165 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02006166 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6167 DataRate::BitsPerSec(kLowBitrateBps),
6168 DataRate::BitsPerSec(kLowBitrateBps),
6169 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006170
6171 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006172 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006173 int64_t timestamp_ms = kFrameIntervalMs;
6174 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6175 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006176 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6177 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006178
6179 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6181 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006182
Artem Titovab30d722021-07-27 16:22:11 +02006183 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006184 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006185 for (size_t i = 1; i <= 10; i++) {
6186 timestamp_ms += kFrameIntervalMs;
6187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6188 WaitForEncodedFrame(timestamp_ms);
6189 }
Åsa Persson06defc42021-09-10 15:28:48 +02006190
6191 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6192 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6193 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6194 timestamp_ms += kFrameIntervalMs;
6195 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6196 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006197 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6198 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6199
Åsa Persson06defc42021-09-10 15:28:48 +02006200 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006201 timestamp_ms += kFrameIntervalMs;
6202 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6203 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006204 // The ramp-up code involves the adaptation queue, give it time to execute.
6205 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006207 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006208
6209 // Frame should not be adapted.
6210 timestamp_ms += kFrameIntervalMs;
6211 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6212 WaitForEncodedFrame(kWidth, kHeight);
6213 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6214
6215 video_stream_encoder_->Stop();
6216}
6217
mflodmancc3d4422017-08-03 08:27:51 -07006218TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006219 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006220 webrtc::test::ScopedFieldTrials field_trials(
6221 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006222 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006223 source.set_adaptation_enabled(true);
6224 video_stream_encoder_->SetSource(&source,
6225 DegradationPreference::MAINTAIN_FRAMERATE);
6226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6227 DataRate::BitsPerSec(kTargetBitrateBps),
6228 DataRate::BitsPerSec(kTargetBitrateBps),
6229 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6230 fake_encoder_.SetQp(kQpHigh + 1);
6231 const int kWidth = 1280;
6232 const int kHeight = 720;
6233 const int64_t kFrameIntervalMs = 100;
6234 int64_t timestamp_ms = kFrameIntervalMs;
6235 for (size_t i = 1; i <= 100; i++) {
6236 timestamp_ms += kFrameIntervalMs;
6237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6238 WaitForEncodedFrame(timestamp_ms);
6239 }
6240 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6241 // for the first time.
6242 // TODO(eshr): We should avoid these waits by using threads with simulated
6243 // time.
6244 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6245 2000 * 2.5 * 2);
6246 timestamp_ms += kFrameIntervalMs;
6247 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6248 WaitForEncodedFrame(timestamp_ms);
6249 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6250 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6251 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6252
6253 // Disable Quality scaling by turning off scaler on the encoder and
6254 // reconfiguring.
6255 fake_encoder_.SetQualityScaling(false);
6256 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6257 kMaxPayloadLength);
6258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006259 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006260 // Since we turned off the quality scaler, the adaptations made by it are
6261 // removed.
6262 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6263 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6264
6265 video_stream_encoder_->Stop();
6266}
6267
6268TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006269 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6270 const int kTooSmallWidth = 10;
6271 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006273 DataRate::BitsPerSec(kTargetBitrateBps),
6274 DataRate::BitsPerSec(kTargetBitrateBps),
6275 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006276
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006277 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006278 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006280 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006281 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006282 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6283
6284 // Trigger adapt down, too small frame, expect no change.
6285 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006286 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006287 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006288 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006289 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6290 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6291
mflodmancc3d4422017-08-03 08:27:51 -07006292 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006293}
6294
mflodmancc3d4422017-08-03 08:27:51 -07006295TEST_F(VideoStreamEncoderTest,
6296 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006297 const int kTooSmallWidth = 10;
6298 const int kTooSmallHeight = 10;
6299 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006300 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006301 DataRate::BitsPerSec(kTargetBitrateBps),
6302 DataRate::BitsPerSec(kTargetBitrateBps),
6303 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006304
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006305 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006306 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006307 video_stream_encoder_->SetSource(&source,
6308 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006309 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006310 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6311 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6312
6313 // Trigger adapt down, expect limited framerate.
6314 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006315 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006316 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006317 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006318 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6319 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6320 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6321
6322 // Trigger adapt down, too small frame, expect no change.
6323 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006324 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006325 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006326 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006327 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6329 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6330
mflodmancc3d4422017-08-03 08:27:51 -07006331 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006332}
6333
mflodmancc3d4422017-08-03 08:27:51 -07006334TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006335 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006337 DataRate::BitsPerSec(kTargetBitrateBps),
6338 DataRate::BitsPerSec(kTargetBitrateBps),
6339 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006340 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006341 const int kFrameWidth = 1280;
6342 const int kFrameHeight = 720;
6343 video_source_.IncomingCapturedFrame(
6344 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006345 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006346 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006347}
6348
sprangb1ca0732017-02-01 08:38:12 -08006349// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006350TEST_F(VideoStreamEncoderTest,
6351 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006353 DataRate::BitsPerSec(kTargetBitrateBps),
6354 DataRate::BitsPerSec(kTargetBitrateBps),
6355 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006356
6357 const int kFrameWidth = 1280;
6358 const int kFrameHeight = 720;
6359 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006360 // requested by
6361 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006362 video_source_.set_adaptation_enabled(true);
6363
6364 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006365 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006366 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006367
6368 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006369 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006370 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006371 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006372 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006373
asaperssonfab67072017-04-04 05:51:49 -07006374 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006375 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006376 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006377 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006378 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006379
mflodmancc3d4422017-08-03 08:27:51 -07006380 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006381}
sprangfe627f32017-03-29 08:24:59 -07006382
mflodmancc3d4422017-08-03 08:27:51 -07006383TEST_F(VideoStreamEncoderTest,
6384 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006385 const int kFrameWidth = 1280;
6386 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006387
Henrik Boström381d1092020-05-12 18:49:07 +02006388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006389 DataRate::BitsPerSec(kTargetBitrateBps),
6390 DataRate::BitsPerSec(kTargetBitrateBps),
6391 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006392 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006393 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006394 video_source_.set_adaptation_enabled(true);
6395
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006396 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006397
6398 video_source_.IncomingCapturedFrame(
6399 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006400 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006401
6402 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006403 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006404
6405 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006406 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006407 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006408 video_source_.IncomingCapturedFrame(
6409 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006410 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006411 }
6412
6413 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006415 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006416 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006417 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006418 video_source_.IncomingCapturedFrame(
6419 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006420 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006421 ++num_frames_dropped;
6422 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006423 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006424 }
6425 }
6426
sprang4847ae62017-06-27 07:06:52 -07006427 // Add some slack to account for frames dropped by the frame dropper.
6428 const int kErrorMargin = 1;
6429 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006430 kErrorMargin);
6431
6432 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006433 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006434 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006435 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006436 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006437 video_source_.IncomingCapturedFrame(
6438 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006439 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006440 ++num_frames_dropped;
6441 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006442 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006443 }
6444 }
sprang4847ae62017-06-27 07:06:52 -07006445 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006446 kErrorMargin);
6447
6448 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006449 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006450 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006451 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006452 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006453 video_source_.IncomingCapturedFrame(
6454 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006455 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006456 ++num_frames_dropped;
6457 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006458 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006459 }
6460 }
sprang4847ae62017-06-27 07:06:52 -07006461 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006462 kErrorMargin);
6463
6464 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006465 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006466 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006467 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006468 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006469 video_source_.IncomingCapturedFrame(
6470 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006471 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006472 ++num_frames_dropped;
6473 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006474 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006475 }
6476 }
6477 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6478
mflodmancc3d4422017-08-03 08:27:51 -07006479 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006480}
6481
mflodmancc3d4422017-08-03 08:27:51 -07006482TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006483 const int kFramerateFps = 5;
6484 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006485 const int kFrameWidth = 1280;
6486 const int kFrameHeight = 720;
6487
sprang4847ae62017-06-27 07:06:52 -07006488 // Reconfigure encoder with two temporal layers and screensharing, which will
6489 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006490 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006491
Henrik Boström381d1092020-05-12 18:49:07 +02006492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006493 DataRate::BitsPerSec(kTargetBitrateBps),
6494 DataRate::BitsPerSec(kTargetBitrateBps),
6495 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006496 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006497 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006498 video_source_.set_adaptation_enabled(true);
6499
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006500 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006501
6502 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006503 rtc::VideoSinkWants last_wants;
6504 do {
6505 last_wants = video_source_.sink_wants();
6506
sprangc5d62e22017-04-02 23:53:04 -07006507 // Insert frames to get a new fps estimate...
6508 for (int j = 0; j < kFramerateFps; ++j) {
6509 video_source_.IncomingCapturedFrame(
6510 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006511 if (video_source_.last_sent_width()) {
6512 sink_.WaitForEncodedFrame(timestamp_ms);
6513 }
sprangc5d62e22017-04-02 23:53:04 -07006514 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006515 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006516 }
6517 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006518 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006519 } while (video_source_.sink_wants().max_framerate_fps <
6520 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006521
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006522 EXPECT_THAT(video_source_.sink_wants(),
6523 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006524
mflodmancc3d4422017-08-03 08:27:51 -07006525 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006526}
asaperssonf7e294d2017-06-13 23:25:22 -07006527
mflodmancc3d4422017-08-03 08:27:51 -07006528TEST_F(VideoStreamEncoderTest,
6529 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006530 const int kWidth = 1280;
6531 const int kHeight = 720;
6532 const int64_t kFrameIntervalMs = 150;
6533 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006535 DataRate::BitsPerSec(kTargetBitrateBps),
6536 DataRate::BitsPerSec(kTargetBitrateBps),
6537 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006538
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006539 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006540 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006541 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006542 video_stream_encoder_->SetSource(&source,
6543 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006544 timestamp_ms += kFrameIntervalMs;
6545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006546 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6550 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6551
6552 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006553 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006554 timestamp_ms += kFrameIntervalMs;
6555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006556 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006557 EXPECT_THAT(source.sink_wants(),
6558 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6561 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6562
6563 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006565 timestamp_ms += kFrameIntervalMs;
6566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006567 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006568 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6571 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
6573 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006574 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006575 timestamp_ms += kFrameIntervalMs;
6576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006577 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006578 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6581 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6582
6583 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006584 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006585 timestamp_ms += kFrameIntervalMs;
6586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006587 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006588 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006589 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6591 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6592
6593 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006594 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006595 timestamp_ms += kFrameIntervalMs;
6596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006597 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006598 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006599 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6601 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6602
6603 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006604 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006605 timestamp_ms += kFrameIntervalMs;
6606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006607 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006608 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6611 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6612
6613 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006614 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006615 timestamp_ms += kFrameIntervalMs;
6616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006617 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006618 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006619 rtc::VideoSinkWants last_wants = source.sink_wants();
6620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6623
6624 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006625 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006626 timestamp_ms += kFrameIntervalMs;
6627 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006628 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006629 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6632 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6633
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006634 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006635 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006636 timestamp_ms += kFrameIntervalMs;
6637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006638 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006639 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6642 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6643
6644 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006645 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006646 timestamp_ms += kFrameIntervalMs;
6647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006648 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006649 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006650 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6652 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6653
6654 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006655 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006656 timestamp_ms += kFrameIntervalMs;
6657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006658 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006659 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6662 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6663
6664 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006665 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006666 timestamp_ms += kFrameIntervalMs;
6667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006668 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006669 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6671 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6672 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6673
6674 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006675 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006676 timestamp_ms += kFrameIntervalMs;
6677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006678 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006679 EXPECT_THAT(source.sink_wants(), FpsMax());
6680 EXPECT_EQ(source.sink_wants().max_pixel_count,
6681 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006682 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6683 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6684 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6685
6686 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006687 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006688 timestamp_ms += kFrameIntervalMs;
6689 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006690 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006691 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006692 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6693 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6694 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6695
Åsa Persson30ab0152019-08-27 12:22:33 +02006696 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006697 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006698 timestamp_ms += kFrameIntervalMs;
6699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006700 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006702 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006703 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6704 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6705 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6706
6707 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006708 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006710 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6711
mflodmancc3d4422017-08-03 08:27:51 -07006712 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006713}
6714
mflodmancc3d4422017-08-03 08:27:51 -07006715TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006716 const int kWidth = 1280;
6717 const int kHeight = 720;
6718 const int64_t kFrameIntervalMs = 150;
6719 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006720 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006721 DataRate::BitsPerSec(kTargetBitrateBps),
6722 DataRate::BitsPerSec(kTargetBitrateBps),
6723 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006724
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006725 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006726 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006727 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006728 video_stream_encoder_->SetSource(&source,
6729 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006730 timestamp_ms += kFrameIntervalMs;
6731 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006732 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006733 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006734 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6737 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6739 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6740
6741 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006742 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006743 timestamp_ms += kFrameIntervalMs;
6744 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006745 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006746 EXPECT_THAT(source.sink_wants(),
6747 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006748 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6750 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6752 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6753 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6754
6755 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006756 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006757 timestamp_ms += kFrameIntervalMs;
6758 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006759 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006761 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6763 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6765 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6766 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6767
6768 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006769 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006770 timestamp_ms += kFrameIntervalMs;
6771 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006772 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006773 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006774 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006775 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6776 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6777 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6778 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6779 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6780
Evan Shrubsole64469032020-06-11 10:45:29 +02006781 // Trigger cpu adapt up, expect no change since QP is most limited.
6782 {
6783 // Store current sink wants since we expect no change and if there is no
6784 // change then last_wants() is not updated.
6785 auto previous_sink_wants = source.sink_wants();
6786 video_stream_encoder_->TriggerCpuUnderuse();
6787 timestamp_ms += kFrameIntervalMs;
6788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6789 WaitForEncodedFrame(timestamp_ms);
6790 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6791 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6792 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6793 }
6794
6795 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6796 video_stream_encoder_->TriggerQualityHigh();
6797 timestamp_ms += kFrameIntervalMs;
6798 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6799 WaitForEncodedFrame(timestamp_ms);
6800 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6801 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6802 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6803 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6804 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6805 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6806 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6807
6808 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6809 // expect increased resolution (960x540@30fps).
6810 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006811 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006812 timestamp_ms += kFrameIntervalMs;
6813 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006814 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006816 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6818 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6819 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6820 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006821 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006822
Evan Shrubsole64469032020-06-11 10:45:29 +02006823 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6824 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006825 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006826 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006827 timestamp_ms += kFrameIntervalMs;
6828 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006829 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006830 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006831 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006832 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6833 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6834 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6835 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6836 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006837 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006838
6839 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006840 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006842 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006843 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006844
mflodmancc3d4422017-08-03 08:27:51 -07006845 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006846}
6847
mflodmancc3d4422017-08-03 08:27:51 -07006848TEST_F(VideoStreamEncoderTest,
6849 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006850 const int kWidth = 640;
6851 const int kHeight = 360;
6852 const int kFpsLimit = 15;
6853 const int64_t kFrameIntervalMs = 150;
6854 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006856 DataRate::BitsPerSec(kTargetBitrateBps),
6857 DataRate::BitsPerSec(kTargetBitrateBps),
6858 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006859
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006860 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006861 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006862 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006863 video_stream_encoder_->SetSource(&source,
6864 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006865 timestamp_ms += kFrameIntervalMs;
6866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006867 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006868 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006869 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6870 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6872 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6873 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6874 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6875
6876 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006877 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006878 timestamp_ms += kFrameIntervalMs;
6879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006880 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006881 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006882 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6884 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6885 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6886 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6887 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6888
6889 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006890 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006891 timestamp_ms += kFrameIntervalMs;
6892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006893 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006894 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006897 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6898 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6899 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6900 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6901
Evan Shrubsole64469032020-06-11 10:45:29 +02006902 // Trigger cpu adapt up, expect no change because quality is most limited.
6903 {
6904 auto previous_sink_wants = source.sink_wants();
6905 // Store current sink wants since we expect no change ind if there is no
6906 // change then last__wants() is not updated.
6907 video_stream_encoder_->TriggerCpuUnderuse();
6908 timestamp_ms += kFrameIntervalMs;
6909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6910 WaitForEncodedFrame(timestamp_ms);
6911 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6912 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6913 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6914 }
6915
6916 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6917 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006918 timestamp_ms += kFrameIntervalMs;
6919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006920 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006921 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006922 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6923 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6924 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006925 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6926 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6927 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006928
Evan Shrubsole64469032020-06-11 10:45:29 +02006929 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006930 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006931 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006932 timestamp_ms += kFrameIntervalMs;
6933 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006934 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006935 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006936 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6937 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6938 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6939 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6940 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006941 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006942
6943 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006944 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006945 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006946 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006947 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006948
mflodmancc3d4422017-08-03 08:27:51 -07006949 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006950}
6951
mflodmancc3d4422017-08-03 08:27:51 -07006952TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006953 const int kFrameWidth = 1920;
6954 const int kFrameHeight = 1080;
6955 // 3/4 of 1920.
6956 const int kAdaptedFrameWidth = 1440;
6957 // 3/4 of 1080 rounded down to multiple of 4.
6958 const int kAdaptedFrameHeight = 808;
6959 const int kFramerate = 24;
6960
Henrik Boström381d1092020-05-12 18:49:07 +02006961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006962 DataRate::BitsPerSec(kTargetBitrateBps),
6963 DataRate::BitsPerSec(kTargetBitrateBps),
6964 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006965 // Trigger reconfigure encoder (without resetting the entire instance).
6966 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006967 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6968 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006969 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006970 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006971 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006972 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006973 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006974 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006975
6976 video_source_.set_adaptation_enabled(true);
6977
6978 video_source_.IncomingCapturedFrame(
6979 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006980 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006981
6982 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006983 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006984 video_source_.IncomingCapturedFrame(
6985 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006986 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006987
mflodmancc3d4422017-08-03 08:27:51 -07006988 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006989}
6990
mflodmancc3d4422017-08-03 08:27:51 -07006991TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006992 const int kFrameWidth = 1280;
6993 const int kFrameHeight = 720;
6994 const int kLowFps = 2;
6995 const int kHighFps = 30;
6996
Henrik Boström381d1092020-05-12 18:49:07 +02006997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006998 DataRate::BitsPerSec(kTargetBitrateBps),
6999 DataRate::BitsPerSec(kTargetBitrateBps),
7000 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07007001
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007002 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007003 max_framerate_ = kLowFps;
7004
7005 // Insert 2 seconds of 2fps video.
7006 for (int i = 0; i < kLowFps * 2; ++i) {
7007 video_source_.IncomingCapturedFrame(
7008 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7009 WaitForEncodedFrame(timestamp_ms);
7010 timestamp_ms += 1000 / kLowFps;
7011 }
7012
7013 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02007014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007015 DataRate::BitsPerSec(kTargetBitrateBps),
7016 DataRate::BitsPerSec(kTargetBitrateBps),
7017 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07007018 video_source_.IncomingCapturedFrame(
7019 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7020 WaitForEncodedFrame(timestamp_ms);
7021 timestamp_ms += 1000 / kLowFps;
7022
7023 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
7024
7025 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02007026 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07007027 const int kFrameIntervalMs = 1000 / kHighFps;
7028 max_framerate_ = kHighFps;
7029 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
7030 video_source_.IncomingCapturedFrame(
7031 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7032 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7033 // be dropped if the encoder hans't been updated with the new higher target
7034 // framerate yet, causing it to overshoot the target bitrate and then
7035 // suffering the wrath of the media optimizer.
7036 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7037 timestamp_ms += kFrameIntervalMs;
7038 }
7039
7040 // Don expect correct measurement just yet, but it should be higher than
7041 // before.
7042 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7043
mflodmancc3d4422017-08-03 08:27:51 -07007044 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007045}
7046
mflodmancc3d4422017-08-03 08:27:51 -07007047TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007048 const int kFrameWidth = 1280;
7049 const int kFrameHeight = 720;
7050 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007051 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007052 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007053 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007054
Henrik Boström381d1092020-05-12 18:49:07 +02007055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007056 DataRate::BitsPerSec(kTargetBitrateBps),
7057 DataRate::BitsPerSec(kTargetBitrateBps),
7058 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007059 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007060
7061 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007062 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007063 video_source_.IncomingCapturedFrame(
7064 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7065 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007066 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007067
7068 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7070 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
7071 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007072
7073 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007074 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007075 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007076
Per Kjellanderdcef6412020-10-07 15:09:05 +02007077 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007078 video_source_.IncomingCapturedFrame(
7079 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7080 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007081 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007082
mflodmancc3d4422017-08-03 08:27:51 -07007083 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007084}
ilnik6b826ef2017-06-16 06:53:48 -07007085
Niels Möller4db138e2018-04-19 09:04:13 +02007086TEST_F(VideoStreamEncoderTest,
7087 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7088 const int kFrameWidth = 1280;
7089 const int kFrameHeight = 720;
7090 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007092 DataRate::BitsPerSec(kTargetBitrateBps),
7093 DataRate::BitsPerSec(kTargetBitrateBps),
7094 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007095 video_source_.IncomingCapturedFrame(
7096 CreateFrame(1, kFrameWidth, kFrameHeight));
7097 WaitForEncodedFrame(1);
7098 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7099 .low_encode_usage_threshold_percent,
7100 default_options.low_encode_usage_threshold_percent);
7101 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7102 .high_encode_usage_threshold_percent,
7103 default_options.high_encode_usage_threshold_percent);
7104 video_stream_encoder_->Stop();
7105}
7106
7107TEST_F(VideoStreamEncoderTest,
7108 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7109 const int kFrameWidth = 1280;
7110 const int kFrameHeight = 720;
7111 CpuOveruseOptions hardware_options;
7112 hardware_options.low_encode_usage_threshold_percent = 150;
7113 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007114 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007115
Henrik Boström381d1092020-05-12 18:49:07 +02007116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007117 DataRate::BitsPerSec(kTargetBitrateBps),
7118 DataRate::BitsPerSec(kTargetBitrateBps),
7119 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007120 video_source_.IncomingCapturedFrame(
7121 CreateFrame(1, kFrameWidth, kFrameHeight));
7122 WaitForEncodedFrame(1);
7123 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7124 .low_encode_usage_threshold_percent,
7125 hardware_options.low_encode_usage_threshold_percent);
7126 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7127 .high_encode_usage_threshold_percent,
7128 hardware_options.high_encode_usage_threshold_percent);
7129 video_stream_encoder_->Stop();
7130}
7131
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007132TEST_F(VideoStreamEncoderTest,
7133 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7134 const int kFrameWidth = 1280;
7135 const int kFrameHeight = 720;
7136
7137 const CpuOveruseOptions default_options;
7138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7139 DataRate::BitsPerSec(kTargetBitrateBps),
7140 DataRate::BitsPerSec(kTargetBitrateBps),
7141 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7142 video_source_.IncomingCapturedFrame(
7143 CreateFrame(1, kFrameWidth, kFrameHeight));
7144 WaitForEncodedFrame(1);
7145 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7146 .low_encode_usage_threshold_percent,
7147 default_options.low_encode_usage_threshold_percent);
7148 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7149 .high_encode_usage_threshold_percent,
7150 default_options.high_encode_usage_threshold_percent);
7151
7152 CpuOveruseOptions hardware_options;
7153 hardware_options.low_encode_usage_threshold_percent = 150;
7154 hardware_options.high_encode_usage_threshold_percent = 200;
7155 fake_encoder_.SetIsHardwareAccelerated(true);
7156
7157 video_source_.IncomingCapturedFrame(
7158 CreateFrame(2, kFrameWidth, kFrameHeight));
7159 WaitForEncodedFrame(2);
7160
7161 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7162 .low_encode_usage_threshold_percent,
7163 hardware_options.low_encode_usage_threshold_percent);
7164 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7165 .high_encode_usage_threshold_percent,
7166 hardware_options.high_encode_usage_threshold_percent);
7167
7168 video_stream_encoder_->Stop();
7169}
7170
Niels Möller6bb5ab92019-01-11 11:11:10 +01007171TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7172 const int kFrameWidth = 320;
7173 const int kFrameHeight = 240;
7174 const int kFps = 30;
7175 const int kTargetBitrateBps = 120000;
7176 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7177
Henrik Boström381d1092020-05-12 18:49:07 +02007178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007179 DataRate::BitsPerSec(kTargetBitrateBps),
7180 DataRate::BitsPerSec(kTargetBitrateBps),
7181 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007182
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007183 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007184 max_framerate_ = kFps;
7185
7186 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7187 fake_encoder_.SimulateOvershoot(1.0);
7188 int num_dropped = 0;
7189 for (int i = 0; i < kNumFramesInRun; ++i) {
7190 video_source_.IncomingCapturedFrame(
7191 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7192 // Wait up to two frame durations for a frame to arrive.
7193 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7194 ++num_dropped;
7195 }
7196 timestamp_ms += 1000 / kFps;
7197 }
7198
Erik Språnga8d48ab2019-02-08 14:17:40 +01007199 // Framerate should be measured to be near the expected target rate.
7200 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7201
7202 // Frame drops should be within 5% of expected 0%.
7203 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007204
7205 // Make encoder produce frames at double the expected bitrate during 3 seconds
7206 // of video, verify number of drops. Rate needs to be slightly changed in
7207 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007208 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007209 const RateControlSettings trials =
7210 RateControlSettings::ParseFromFieldTrials();
7211 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007212 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007213 // frame dropping since the adjuter will try to just lower the target
7214 // bitrate rather than drop frames. If network headroom can be used, it
7215 // doesn't push back as hard so we don't need quite as much overshoot.
7216 // These numbers are unfortunately a bit magical but there's not trivial
7217 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007218 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007219 }
7220 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007222 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7223 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
7224 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007225 num_dropped = 0;
7226 for (int i = 0; i < kNumFramesInRun; ++i) {
7227 video_source_.IncomingCapturedFrame(
7228 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7229 // Wait up to two frame durations for a frame to arrive.
7230 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7231 ++num_dropped;
7232 }
7233 timestamp_ms += 1000 / kFps;
7234 }
7235
Henrik Boström381d1092020-05-12 18:49:07 +02007236 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007237 DataRate::BitsPerSec(kTargetBitrateBps),
7238 DataRate::BitsPerSec(kTargetBitrateBps),
7239 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007240
7241 // Target framerate should be still be near the expected target, despite
7242 // the frame drops.
7243 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7244
7245 // Frame drops should be within 5% of expected 50%.
7246 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007247
7248 video_stream_encoder_->Stop();
7249}
7250
7251TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7252 const int kFrameWidth = 320;
7253 const int kFrameHeight = 240;
7254 const int kActualInputFps = 24;
7255 const int kTargetBitrateBps = 120000;
7256
7257 ASSERT_GT(max_framerate_, kActualInputFps);
7258
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007259 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007260 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007262 DataRate::BitsPerSec(kTargetBitrateBps),
7263 DataRate::BitsPerSec(kTargetBitrateBps),
7264 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007265
7266 // Insert 3 seconds of video, with an input fps lower than configured max.
7267 for (int i = 0; i < kActualInputFps * 3; ++i) {
7268 video_source_.IncomingCapturedFrame(
7269 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7270 // Wait up to two frame durations for a frame to arrive.
7271 WaitForEncodedFrame(timestamp_ms);
7272 timestamp_ms += 1000 / kActualInputFps;
7273 }
7274
7275 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7276
7277 video_stream_encoder_->Stop();
7278}
7279
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007280TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007281 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007282 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007283 DataRate::BitsPerSec(kTargetBitrateBps),
7284 DataRate::BitsPerSec(kTargetBitrateBps),
7285 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007286
7287 fake_encoder_.BlockNextEncode();
7288 video_source_.IncomingCapturedFrame(
7289 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7290 WaitForEncodedFrame(1);
7291 // On the very first frame full update should be forced.
7292 rect = fake_encoder_.GetLastUpdateRect();
7293 EXPECT_EQ(rect.offset_x, 0);
7294 EXPECT_EQ(rect.offset_y, 0);
7295 EXPECT_EQ(rect.height, codec_height_);
7296 EXPECT_EQ(rect.width, codec_width_);
7297 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7298 // call to ContinueEncode.
7299 video_source_.IncomingCapturedFrame(
7300 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7301 ExpectDroppedFrame();
7302 video_source_.IncomingCapturedFrame(
7303 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7304 ExpectDroppedFrame();
7305 fake_encoder_.ContinueEncode();
7306 WaitForEncodedFrame(3);
7307 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7308 rect = fake_encoder_.GetLastUpdateRect();
7309 EXPECT_EQ(rect.offset_x, 1);
7310 EXPECT_EQ(rect.offset_y, 0);
7311 EXPECT_EQ(rect.width, 10);
7312 EXPECT_EQ(rect.height, 1);
7313
7314 video_source_.IncomingCapturedFrame(
7315 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7316 WaitForEncodedFrame(4);
7317 // Previous frame was encoded, so no accumulation should happen.
7318 rect = fake_encoder_.GetLastUpdateRect();
7319 EXPECT_EQ(rect.offset_x, 0);
7320 EXPECT_EQ(rect.offset_y, 0);
7321 EXPECT_EQ(rect.width, 1);
7322 EXPECT_EQ(rect.height, 1);
7323
7324 video_stream_encoder_->Stop();
7325}
7326
Erik Språngd7329ca2019-02-21 21:19:53 +01007327TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007329 DataRate::BitsPerSec(kTargetBitrateBps),
7330 DataRate::BitsPerSec(kTargetBitrateBps),
7331 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007332
7333 // First frame is always keyframe.
7334 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7335 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007336 EXPECT_THAT(
7337 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007338 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007339
7340 // Insert delta frame.
7341 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7342 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007343 EXPECT_THAT(
7344 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007345 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007346
7347 // Request next frame be a key-frame.
7348 video_stream_encoder_->SendKeyFrame();
7349 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7350 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007351 EXPECT_THAT(
7352 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007353 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007354
7355 video_stream_encoder_->Stop();
7356}
7357
7358TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7359 // Setup simulcast with three streams.
7360 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007362 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7363 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7364 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007365 // Wait for all three layers before triggering event.
7366 sink_.SetNumExpectedLayers(3);
7367
7368 // First frame is always keyframe.
7369 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7370 WaitForEncodedFrame(1);
7371 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007372 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7373 VideoFrameType::kVideoFrameKey,
7374 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007375
7376 // Insert delta frame.
7377 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7378 WaitForEncodedFrame(2);
7379 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007380 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7381 VideoFrameType::kVideoFrameDelta,
7382 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007383
7384 // Request next frame be a key-frame.
7385 // Only first stream is configured to produce key-frame.
7386 video_stream_encoder_->SendKeyFrame();
7387 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7388 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007389
7390 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7391 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007392 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007393 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007394 VideoFrameType::kVideoFrameKey,
7395 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007396
7397 video_stream_encoder_->Stop();
7398}
7399
7400TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7401 // Configure internal source factory and setup test again.
7402 encoder_factory_.SetHasInternalSource(true);
7403 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007404 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007405 DataRate::BitsPerSec(kTargetBitrateBps),
7406 DataRate::BitsPerSec(kTargetBitrateBps),
7407 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007408
7409 // Call encoder directly, simulating internal source where encoded frame
7410 // callback in VideoStreamEncoder is called despite no OnFrame().
7411 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7412 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007413 EXPECT_THAT(
7414 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007415 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007416
Niels Möller8f7ce222019-03-21 15:43:58 +01007417 const std::vector<VideoFrameType> kDeltaFrame = {
7418 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007419 // Need to set timestamp manually since manually for injected frame.
7420 VideoFrame frame = CreateFrame(101, nullptr);
7421 frame.set_timestamp(101);
7422 fake_encoder_.InjectFrame(frame, false);
7423 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007424 EXPECT_THAT(
7425 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007426 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007427
7428 // Request key-frame. The forces a dummy frame down into the encoder.
7429 fake_encoder_.ExpectNullFrame();
7430 video_stream_encoder_->SendKeyFrame();
7431 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007432 EXPECT_THAT(
7433 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007434 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007435
7436 video_stream_encoder_->Stop();
7437}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007438
7439TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7440 // Configure internal source factory and setup test again.
7441 encoder_factory_.SetHasInternalSource(true);
7442 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007444 DataRate::BitsPerSec(kTargetBitrateBps),
7445 DataRate::BitsPerSec(kTargetBitrateBps),
7446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007447
7448 int64_t timestamp = 1;
7449 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007450 image.capture_time_ms_ = ++timestamp;
7451 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7452 const int64_t kEncodeFinishDelayMs = 10;
7453 image.timing_.encode_start_ms = timestamp;
7454 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007455 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007456 // Wait for frame without incrementing clock.
7457 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7458 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7459 // capture timestamp should be kEncodeFinishDelayMs in the past.
7460 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007461 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007462
7463 video_stream_encoder_->Stop();
7464}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007465
7466TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007467 // SPS contains VUI with restrictions on the maximum number of reordered
7468 // pictures, there is no need to rewrite the bitstream to enable faster
7469 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007470 ResetEncoder("H264", 1, 1, 1, false);
7471
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7473 DataRate::BitsPerSec(kTargetBitrateBps),
7474 DataRate::BitsPerSec(kTargetBitrateBps),
7475 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7476 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007477
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007478 fake_encoder_.SetEncodedImageData(
7479 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007480
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007481 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7482 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007483
7484 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7485 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007486
7487 video_stream_encoder_->Stop();
7488}
7489
7490TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007491 // SPS does not contain VUI, the bitstream is will be rewritten with added
7492 // VUI with restrictions on the maximum number of reordered pictures to
7493 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007494 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7495 0x00, 0x00, 0x03, 0x03, 0xF4,
7496 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007497 ResetEncoder("H264", 1, 1, 1, false);
7498
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7500 DataRate::BitsPerSec(kTargetBitrateBps),
7501 DataRate::BitsPerSec(kTargetBitrateBps),
7502 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7503 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007504
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007505 fake_encoder_.SetEncodedImageData(
7506 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007507
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007508 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7509 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007510
7511 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7512 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007513
7514 video_stream_encoder_->Stop();
7515}
7516
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007517TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7518 const int kFrameWidth = 1280;
7519 const int kFrameHeight = 720;
7520 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7521
Henrik Boström381d1092020-05-12 18:49:07 +02007522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007523 DataRate::BitsPerSec(kTargetBitrateBps),
7524 DataRate::BitsPerSec(kTargetBitrateBps),
7525 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007526 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7527
7528 // Insert a first video frame. It should be dropped because of downscale in
7529 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007530 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007531 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7532 frame.set_rotation(kVideoRotation_270);
7533 video_source_.IncomingCapturedFrame(frame);
7534
7535 ExpectDroppedFrame();
7536
7537 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007538 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007539 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7540 frame.set_rotation(kVideoRotation_90);
7541 video_source_.IncomingCapturedFrame(frame);
7542
7543 WaitForEncodedFrame(timestamp_ms);
7544 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7545
7546 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007547 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007548 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7549 frame.set_rotation(kVideoRotation_180);
7550 video_source_.IncomingCapturedFrame(frame);
7551
7552 WaitForEncodedFrame(timestamp_ms);
7553 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7554
7555 video_stream_encoder_->Stop();
7556}
7557
Erik Språng5056af02019-09-02 15:53:11 +02007558TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7559 const int kFrameWidth = 320;
7560 const int kFrameHeight = 180;
7561
7562 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007564 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7565 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7566 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007567 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007568 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007569 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007570
7571 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007572 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007573 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7574 frame.set_rotation(kVideoRotation_270);
7575 video_source_.IncomingCapturedFrame(frame);
7576 WaitForEncodedFrame(timestamp_ms);
7577
7578 // Set a target rate below the minimum allowed by the codec settings.
7579 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007580 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7581 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007583 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007584 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007585 /*link_allocation=*/target_rate,
7586 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007587 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007588 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007589 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7590
7591 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7592 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7593 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007594 DataRate allocation_sum =
7595 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007596 EXPECT_EQ(min_rate, allocation_sum);
7597 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7598
7599 video_stream_encoder_->Stop();
7600}
7601
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007602TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007604 DataRate::BitsPerSec(kTargetBitrateBps),
7605 DataRate::BitsPerSec(kTargetBitrateBps),
7606 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007607 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007608 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007609 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7610 WaitForEncodedFrame(1);
7611
7612 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7613 ASSERT_TRUE(prev_rate_settings.has_value());
7614 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7615 kDefaultFramerate);
7616
7617 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7618 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7619 timestamp_ms += 1000 / kDefaultFramerate;
7620 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7621 WaitForEncodedFrame(timestamp_ms);
7622 }
7623 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7624 kDefaultFramerate);
7625 // Capture larger frame to trigger a reconfigure.
7626 codec_height_ *= 2;
7627 codec_width_ *= 2;
7628 timestamp_ms += 1000 / kDefaultFramerate;
7629 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7630 WaitForEncodedFrame(timestamp_ms);
7631
7632 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7633 auto current_rate_settings =
7634 fake_encoder_.GetAndResetLastRateControlSettings();
7635 // Ensure we have actually reconfigured twice
7636 // The rate settings should have been set again even though
7637 // they haven't changed.
7638 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007639 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007640
7641 video_stream_encoder_->Stop();
7642}
7643
philipeld9cc8c02019-09-16 14:53:40 +02007644struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007645 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7646 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7647 MOCK_METHOD(void,
7648 RequestEncoderSwitch,
7649 (const webrtc::SdpVideoFormat& format),
7650 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007651};
7652
philipel9b058032020-02-10 11:30:00 +01007653TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7654 constexpr int kDontCare = 100;
7655 StrictMock<MockEncoderSelector> encoder_selector;
7656 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7657 &fake_encoder_, &encoder_selector);
7658 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7659
7660 // Reset encoder for new configuration to take effect.
7661 ConfigureEncoder(video_encoder_config_.Copy());
7662
7663 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7664
7665 video_source_.IncomingCapturedFrame(
7666 CreateFrame(kDontCare, kDontCare, kDontCare));
7667 video_stream_encoder_->Stop();
7668
7669 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7670 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007671 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7672 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007673 video_stream_encoder_.reset();
7674}
7675
7676TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7677 constexpr int kDontCare = 100;
7678
7679 NiceMock<MockEncoderSelector> encoder_selector;
7680 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7681 video_send_config_.encoder_settings.encoder_switch_request_callback =
7682 &switch_callback;
7683 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7684 &fake_encoder_, &encoder_selector);
7685 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7686
7687 // Reset encoder for new configuration to take effect.
7688 ConfigureEncoder(video_encoder_config_.Copy());
7689
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007690 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007691 .WillByDefault(Return(SdpVideoFormat("AV1")));
7692 EXPECT_CALL(switch_callback,
7693 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7694 Field(&SdpVideoFormat::name, "AV1"))));
7695
Henrik Boström381d1092020-05-12 18:49:07 +02007696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007697 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7698 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7699 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007700 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007701 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007702 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007703 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007704
7705 video_stream_encoder_->Stop();
7706}
7707
7708TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7709 constexpr int kSufficientBitrateToNotDrop = 1000;
7710 constexpr int kDontCare = 100;
7711
7712 NiceMock<MockVideoEncoder> video_encoder;
7713 NiceMock<MockEncoderSelector> encoder_selector;
7714 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7715 video_send_config_.encoder_settings.encoder_switch_request_callback =
7716 &switch_callback;
7717 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7718 &video_encoder, &encoder_selector);
7719 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7720
7721 // Reset encoder for new configuration to take effect.
7722 ConfigureEncoder(video_encoder_config_.Copy());
7723
7724 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7725 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7726 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007728 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7729 /*stable_target_bitrate=*/
7730 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7731 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007732 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007733 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007734 /*cwnd_reduce_ratio=*/0);
7735
7736 ON_CALL(video_encoder, Encode(_, _))
7737 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7738 ON_CALL(encoder_selector, OnEncoderBroken())
7739 .WillByDefault(Return(SdpVideoFormat("AV2")));
7740
7741 rtc::Event encode_attempted;
7742 EXPECT_CALL(switch_callback,
7743 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7744 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7745 EXPECT_EQ(format.name, "AV2");
7746 encode_attempted.Set();
7747 });
7748
7749 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7750 encode_attempted.Wait(3000);
7751
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007752 AdvanceTime(TimeDelta::Millis(0));
7753
philipel9b058032020-02-10 11:30:00 +01007754 video_stream_encoder_->Stop();
7755
7756 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7757 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007758 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7759 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007760 video_stream_encoder_.reset();
7761}
7762
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007763TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007764 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007765 const int kFrameWidth = 320;
7766 const int kFrameHeight = 180;
7767
7768 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007769 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007771 /*target_bitrate=*/rate,
7772 /*stable_target_bitrate=*/rate,
7773 /*link_allocation=*/rate,
7774 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007775 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007776 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007777
7778 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007779 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007780 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7781 frame.set_rotation(kVideoRotation_270);
7782 video_source_.IncomingCapturedFrame(frame);
7783 WaitForEncodedFrame(timestamp_ms);
7784 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7785
7786 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007787 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007789 /*target_bitrate=*/new_stable_rate,
7790 /*stable_target_bitrate=*/new_stable_rate,
7791 /*link_allocation=*/rate,
7792 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007793 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007794 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007795 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7796 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7797 video_stream_encoder_->Stop();
7798}
7799
7800TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007801 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007802 const int kFrameWidth = 320;
7803 const int kFrameHeight = 180;
7804
7805 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007806 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007808 /*target_bitrate=*/rate,
7809 /*stable_target_bitrate=*/rate,
7810 /*link_allocation=*/rate,
7811 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007812 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007813 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007814
7815 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007816 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007817 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7818 frame.set_rotation(kVideoRotation_270);
7819 video_source_.IncomingCapturedFrame(frame);
7820 WaitForEncodedFrame(timestamp_ms);
7821 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7822
7823 // Set a higher target rate without changing the link_allocation. Should not
7824 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007825 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007827 /*target_bitrate=*/rate,
7828 /*stable_target_bitrate=*/new_stable_rate,
7829 /*link_allocation=*/rate,
7830 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007831 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007832 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007833 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7834 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7835 video_stream_encoder_->Stop();
7836}
7837
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007838TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7839 test::ScopedFieldTrials field_trials(
7840 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7841 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7842 const int kFramerateFps = 30;
7843 const int kWidth = 1920;
7844 const int kHeight = 1080;
7845 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7846 // Works on screenshare mode.
7847 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7848 // We rely on the automatic resolution adaptation, but we handle framerate
7849 // adaptation manually by mocking the stats proxy.
7850 video_source_.set_adaptation_enabled(true);
7851
7852 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007854 DataRate::BitsPerSec(kTargetBitrateBps),
7855 DataRate::BitsPerSec(kTargetBitrateBps),
7856 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007857 video_stream_encoder_->SetSource(&video_source_,
7858 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007859 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007860
7861 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7862 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7863
7864 // Pass enough frames with the full update to trigger animation detection.
7865 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007866 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007867 frame.set_ntp_time_ms(timestamp_ms);
7868 frame.set_timestamp_us(timestamp_ms * 1000);
7869 video_source_.IncomingCapturedFrame(frame);
7870 WaitForEncodedFrame(timestamp_ms);
7871 }
7872
7873 // Resolution should be limited.
7874 rtc::VideoSinkWants expected;
7875 expected.max_framerate_fps = kFramerateFps;
7876 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007877 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007878
7879 // Pass one frame with no known update.
7880 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007881 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007882 frame.set_ntp_time_ms(timestamp_ms);
7883 frame.set_timestamp_us(timestamp_ms * 1000);
7884 frame.clear_update_rect();
7885
7886 video_source_.IncomingCapturedFrame(frame);
7887 WaitForEncodedFrame(timestamp_ms);
7888
7889 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007890 EXPECT_THAT(video_source_.sink_wants(),
7891 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007892
7893 video_stream_encoder_->Stop();
7894}
7895
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007896TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7897 const int kWidth = 720; // 540p adapted down.
7898 const int kHeight = 405;
7899 const int kNumFrames = 3;
7900 // Works on screenshare mode.
7901 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7902 /*num_spatial_layers=*/2, /*screenshare=*/true);
7903
7904 video_source_.set_adaptation_enabled(true);
7905
7906 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7907 DataRate::BitsPerSec(kTargetBitrateBps),
7908 DataRate::BitsPerSec(kTargetBitrateBps),
7909 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7910
7911 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7912
7913 // Pass enough frames with the full update to trigger animation detection.
7914 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007915 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007916 frame.set_ntp_time_ms(timestamp_ms);
7917 frame.set_timestamp_us(timestamp_ms * 1000);
7918 video_source_.IncomingCapturedFrame(frame);
7919 WaitForEncodedFrame(timestamp_ms);
7920 }
7921
7922 video_stream_encoder_->Stop();
7923}
7924
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007925TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7926 const float downscale_factors[] = {4.0, 2.0, 1.0};
7927 const int number_layers =
7928 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7929 VideoEncoderConfig config;
7930 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7931 for (int i = 0; i < number_layers; ++i) {
7932 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7933 config.simulcast_layers[i].active = true;
7934 }
7935 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007936 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007937 "VP8", /*max qp*/ 56, /*screencast*/ false,
7938 /*screenshare enabled*/ false);
7939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7940 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7941 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7942 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7943
7944 // First initialization.
7945 // Encoder should be initialized. Next frame should be key frame.
7946 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7947 sink_.SetNumExpectedLayers(number_layers);
7948 int64_t timestamp_ms = kFrameIntervalMs;
7949 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7950 WaitForEncodedFrame(timestamp_ms);
7951 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7952 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7953 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7954 VideoFrameType::kVideoFrameKey,
7955 VideoFrameType::kVideoFrameKey}));
7956
7957 // Disable top layer.
7958 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7959 config.simulcast_layers[number_layers - 1].active = false;
7960 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7961 sink_.SetNumExpectedLayers(number_layers - 1);
7962 timestamp_ms += kFrameIntervalMs;
7963 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7964 WaitForEncodedFrame(timestamp_ms);
7965 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7966 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7967 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7968 VideoFrameType::kVideoFrameDelta,
7969 VideoFrameType::kVideoFrameDelta}));
7970
7971 // Re-enable top layer.
7972 // Encoder should be re-initialized. Next frame should be key frame.
7973 config.simulcast_layers[number_layers - 1].active = true;
7974 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7975 sink_.SetNumExpectedLayers(number_layers);
7976 timestamp_ms += kFrameIntervalMs;
7977 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7978 WaitForEncodedFrame(timestamp_ms);
7979 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7980 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7981 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7982 VideoFrameType::kVideoFrameKey,
7983 VideoFrameType::kVideoFrameKey}));
7984
7985 // Top layer max rate change.
7986 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7987 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7988 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7989 sink_.SetNumExpectedLayers(number_layers);
7990 timestamp_ms += kFrameIntervalMs;
7991 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7992 WaitForEncodedFrame(timestamp_ms);
7993 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7994 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7995 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7996 VideoFrameType::kVideoFrameDelta,
7997 VideoFrameType::kVideoFrameDelta}));
7998
7999 // Top layer resolution change.
8000 // Encoder should be re-initialized. Next frame should be key frame.
8001 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8002 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8003 sink_.SetNumExpectedLayers(number_layers);
8004 timestamp_ms += kFrameIntervalMs;
8005 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8006 WaitForEncodedFrame(timestamp_ms);
8007 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
8008 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8009 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8010 VideoFrameType::kVideoFrameKey,
8011 VideoFrameType::kVideoFrameKey}));
8012 video_stream_encoder_->Stop();
8013}
8014
Henrik Boström1124ed12021-02-25 10:30:39 +01008015TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8016 const int kFrameWidth = 1280;
8017 const int kFrameHeight = 720;
8018
8019 SetUp();
8020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8021 DataRate::BitsPerSec(kTargetBitrateBps),
8022 DataRate::BitsPerSec(kTargetBitrateBps),
8023 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8024
8025 // Capturing a frame should reconfigure the encoder and expose the encoder
8026 // resolution, which is the same as the input frame.
8027 int64_t timestamp_ms = kFrameIntervalMs;
8028 video_source_.IncomingCapturedFrame(
8029 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8030 WaitForEncodedFrame(timestamp_ms);
8031 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8032 EXPECT_THAT(video_source_.sink_wants().resolutions,
8033 ::testing::ElementsAreArray(
8034 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8035
8036 video_stream_encoder_->Stop();
8037}
8038
8039TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8040 // Pick downscale factors such that we never encode at full resolution - this
8041 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008042 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008043 // encoder should not ask for the frame resolution. This allows video frames
8044 // to have the appearence of one resolution but optimize its internal buffers
8045 // for what is actually encoded.
8046 const size_t kNumSimulcastLayers = 3u;
8047 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8048 const int kFrameWidth = 1280;
8049 const int kFrameHeight = 720;
8050 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8051 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8052 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8053 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8054 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8055 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8056
8057 VideoEncoderConfig config;
8058 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8059 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8060 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8061 config.simulcast_layers[i].active = true;
8062 }
8063 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008064 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008065 "VP8", /*max qp*/ 56, /*screencast*/ false,
8066 /*screenshare enabled*/ false);
8067 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8068 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8069 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8070 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8071
8072 // Capture a frame with all layers active.
8073 int64_t timestamp_ms = kFrameIntervalMs;
8074 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8075 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8076 video_source_.IncomingCapturedFrame(
8077 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8078 WaitForEncodedFrame(timestamp_ms);
8079 // Expect encoded resolutions to match the expected simulcast layers.
8080 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8081 EXPECT_THAT(
8082 video_source_.sink_wants().resolutions,
8083 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8084
8085 // Capture a frame with one of the layers inactive.
8086 timestamp_ms += kFrameIntervalMs;
8087 config.simulcast_layers[2].active = false;
8088 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8089 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8090 video_source_.IncomingCapturedFrame(
8091 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8092 WaitForEncodedFrame(timestamp_ms);
8093
8094 // Expect encoded resolutions to match the expected simulcast layers.
8095 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8096 EXPECT_THAT(video_source_.sink_wants().resolutions,
8097 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8098
8099 // Capture a frame with all but one layer turned off.
8100 timestamp_ms += kFrameIntervalMs;
8101 config.simulcast_layers[1].active = false;
8102 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8103 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8104 video_source_.IncomingCapturedFrame(
8105 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8106 WaitForEncodedFrame(timestamp_ms);
8107
8108 // Expect encoded resolutions to match the expected simulcast layers.
8109 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8110 EXPECT_THAT(video_source_.sink_wants().resolutions,
8111 ::testing::ElementsAreArray({kLayer0Size}));
8112
8113 video_stream_encoder_->Stop();
8114}
8115
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008116TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008117 ResetEncoder("VP8", 1, 1, 1, false);
8118
Niels Möller8b692902021-06-14 12:04:57 +02008119 // Force encoder reconfig.
8120 video_source_.IncomingCapturedFrame(
8121 CreateFrame(1, codec_width_, codec_height_));
8122 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8123
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008124 // Set QP on encoded frame and pass the frame to encode complete callback.
8125 // Since QP is present QP parsing won't be triggered and the original value
8126 // should be kept.
8127 EncodedImage encoded_image;
8128 encoded_image.qp_ = 123;
8129 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8130 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8131 CodecSpecificInfo codec_info;
8132 codec_info.codecType = kVideoCodecVP8;
8133 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8134 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8135 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8136 video_stream_encoder_->Stop();
8137}
8138
8139TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008140 ResetEncoder("VP8", 1, 1, 1, false);
8141
Niels Möller8b692902021-06-14 12:04:57 +02008142 // Force encoder reconfig.
8143 video_source_.IncomingCapturedFrame(
8144 CreateFrame(1, codec_width_, codec_height_));
8145 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8146
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008147 // Pass an encoded frame without QP to encode complete callback. QP should be
8148 // parsed and set.
8149 EncodedImage encoded_image;
8150 encoded_image.qp_ = -1;
8151 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8152 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8153 CodecSpecificInfo codec_info;
8154 codec_info.codecType = kVideoCodecVP8;
8155 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8156 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8157 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8158 video_stream_encoder_->Stop();
8159}
8160
8161TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
8162 webrtc::test::ScopedFieldTrials field_trials(
8163 "WebRTC-QpParsingKillSwitch/Enabled/");
8164
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008165 ResetEncoder("VP8", 1, 1, 1, false);
8166
Niels Möller8b692902021-06-14 12:04:57 +02008167 // Force encoder reconfig.
8168 video_source_.IncomingCapturedFrame(
8169 CreateFrame(1, codec_width_, codec_height_));
8170 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8171
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008172 EncodedImage encoded_image;
8173 encoded_image.qp_ = -1;
8174 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8175 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8176 CodecSpecificInfo codec_info;
8177 codec_info.codecType = kVideoCodecVP8;
8178 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8179 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8180 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8181 video_stream_encoder_->Stop();
8182}
8183
Sergey Silkind19e3b92021-03-16 10:05:30 +00008184TEST_F(VideoStreamEncoderTest,
8185 QualityScalingNotAllowed_QualityScalingDisabled) {
8186 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8187
8188 // Disable scaling settings in encoder info.
8189 fake_encoder_.SetQualityScaling(false);
8190 // Disable quality scaling in encoder config.
8191 video_encoder_config.is_quality_scaling_allowed = false;
8192 ConfigureEncoder(std::move(video_encoder_config));
8193
8194 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8195 DataRate::BitsPerSec(kTargetBitrateBps),
8196 DataRate::BitsPerSec(kTargetBitrateBps),
8197 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8198
8199 test::FrameForwarder source;
8200 video_stream_encoder_->SetSource(
8201 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8202 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8203 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8204
8205 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8206 WaitForEncodedFrame(1);
8207 video_stream_encoder_->TriggerQualityLow();
8208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8209
8210 video_stream_encoder_->Stop();
8211}
8212
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008213TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8214 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8215
8216 // Disable scaling settings in encoder info.
8217 fake_encoder_.SetQualityScaling(false);
8218 // Set QP not trusted in encoder info.
8219 fake_encoder_.SetIsQpTrusted(false);
8220 // Enable quality scaling in encoder config.
8221 video_encoder_config.is_quality_scaling_allowed = true;
8222 ConfigureEncoder(std::move(video_encoder_config));
8223
8224 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8225 DataRate::BitsPerSec(kTargetBitrateBps),
8226 DataRate::BitsPerSec(kTargetBitrateBps),
8227 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8228
8229 test::FrameForwarder source;
8230 video_stream_encoder_->SetSource(
8231 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8232 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8234
8235 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8236 WaitForEncodedFrame(1);
8237 video_stream_encoder_->TriggerQualityLow();
8238 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8239
8240 video_stream_encoder_->Stop();
8241}
8242
8243TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8244 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8245
8246 // Disable scaling settings in encoder info.
8247 fake_encoder_.SetQualityScaling(false);
8248 // Set QP trusted in encoder info.
8249 fake_encoder_.SetIsQpTrusted(true);
8250 // Enable quality scaling in encoder config.
8251 video_encoder_config.is_quality_scaling_allowed = false;
8252 ConfigureEncoder(std::move(video_encoder_config));
8253
8254 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8255 DataRate::BitsPerSec(kTargetBitrateBps),
8256 DataRate::BitsPerSec(kTargetBitrateBps),
8257 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8258
8259 test::FrameForwarder source;
8260 video_stream_encoder_->SetSource(
8261 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8262 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8263 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8264
8265 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8266 WaitForEncodedFrame(1);
8267 video_stream_encoder_->TriggerQualityLow();
8268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8269
8270 video_stream_encoder_->Stop();
8271}
8272
Sergey Silkind19e3b92021-03-16 10:05:30 +00008273#if !defined(WEBRTC_IOS)
8274// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8275// disabled by default on iOS.
8276TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8277 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8278
8279 // Disable scaling settings in encoder info.
8280 fake_encoder_.SetQualityScaling(false);
8281 // Enable quality scaling in encoder config.
8282 video_encoder_config.is_quality_scaling_allowed = true;
8283 ConfigureEncoder(std::move(video_encoder_config));
8284
8285 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8286 DataRate::BitsPerSec(kTargetBitrateBps),
8287 DataRate::BitsPerSec(kTargetBitrateBps),
8288 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8289
8290 test::FrameForwarder source;
8291 video_stream_encoder_->SetSource(
8292 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8293 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8295
8296 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8297 WaitForEncodedFrame(1);
8298 video_stream_encoder_->TriggerQualityLow();
8299 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8300
8301 video_stream_encoder_->Stop();
8302}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008303
8304TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8305 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8306
8307 // Disable scaling settings in encoder info.
8308 fake_encoder_.SetQualityScaling(false);
8309 // Set QP trusted in encoder info.
8310 fake_encoder_.SetIsQpTrusted(true);
8311 // Enable quality scaling in encoder config.
8312 video_encoder_config.is_quality_scaling_allowed = true;
8313 ConfigureEncoder(std::move(video_encoder_config));
8314
8315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8316 DataRate::BitsPerSec(kTargetBitrateBps),
8317 DataRate::BitsPerSec(kTargetBitrateBps),
8318 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8319
8320 test::FrameForwarder source;
8321 video_stream_encoder_->SetSource(
8322 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8323 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8324 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8325
8326 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8327 WaitForEncodedFrame(1);
8328 video_stream_encoder_->TriggerQualityLow();
8329 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8330
8331 video_stream_encoder_->Stop();
8332}
Sergey Silkind19e3b92021-03-16 10:05:30 +00008333#endif
8334
Henrik Boström56db9ff2021-03-24 09:06:45 +01008335// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8336class VideoStreamEncoderWithRealEncoderTest
8337 : public VideoStreamEncoderTest,
8338 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8339 public:
8340 VideoStreamEncoderWithRealEncoderTest()
8341 : VideoStreamEncoderTest(),
8342 codec_type_(std::get<0>(GetParam())),
8343 allow_i420_conversion_(std::get<1>(GetParam())) {}
8344
8345 void SetUp() override {
8346 VideoStreamEncoderTest::SetUp();
8347 std::unique_ptr<VideoEncoder> encoder;
8348 switch (codec_type_) {
8349 case kVideoCodecVP8:
8350 encoder = VP8Encoder::Create();
8351 break;
8352 case kVideoCodecVP9:
8353 encoder = VP9Encoder::Create();
8354 break;
8355 case kVideoCodecAV1:
8356 encoder = CreateLibaomAv1Encoder();
8357 break;
8358 case kVideoCodecH264:
8359 encoder =
8360 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8361 break;
8362 case kVideoCodecMultiplex:
8363 mock_encoder_factory_for_multiplex_ =
8364 std::make_unique<MockVideoEncoderFactory>();
8365 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8366 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8367 .WillRepeatedly([] { return VP8Encoder::Create(); });
8368 encoder = std::make_unique<MultiplexEncoderAdapter>(
8369 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8370 false);
8371 break;
8372 default:
8373 RTC_NOTREACHED();
8374 }
8375 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8376 }
8377
8378 void TearDown() override {
8379 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008380 // Ensure `video_stream_encoder_` is destroyed before
8381 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008382 video_stream_encoder_.reset();
8383 VideoStreamEncoderTest::TearDown();
8384 }
8385
8386 protected:
8387 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8388 std::unique_ptr<VideoEncoder> encoder) {
8389 // Configure VSE to use the encoder.
8390 encoder_ = std::move(encoder);
8391 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8392 encoder_.get(), &encoder_selector_);
8393 video_send_config_.encoder_settings.encoder_factory =
8394 encoder_proxy_factory_.get();
8395 VideoEncoderConfig video_encoder_config;
8396 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8397 video_encoder_config_ = video_encoder_config.Copy();
8398 ConfigureEncoder(video_encoder_config_.Copy());
8399
8400 // Set bitrate to ensure frame is not dropped.
8401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8402 DataRate::BitsPerSec(kTargetBitrateBps),
8403 DataRate::BitsPerSec(kTargetBitrateBps),
8404 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8405 }
8406
8407 const VideoCodecType codec_type_;
8408 const bool allow_i420_conversion_;
8409 NiceMock<MockEncoderSelector> encoder_selector_;
8410 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8411 std::unique_ptr<VideoEncoder> encoder_;
8412 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8413};
8414
8415TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8416 auto native_i420_frame = test::CreateMappableNativeFrame(
8417 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8418 video_source_.IncomingCapturedFrame(native_i420_frame);
8419 WaitForEncodedFrame(codec_width_, codec_height_);
8420
8421 auto mappable_native_buffer =
8422 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8423 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8424 mappable_native_buffer->GetMappedFramedBuffers();
8425 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8426 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8427 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8428 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8429}
8430
8431TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8432 auto native_nv12_frame = test::CreateMappableNativeFrame(
8433 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8434 video_source_.IncomingCapturedFrame(native_nv12_frame);
8435 WaitForEncodedFrame(codec_width_, codec_height_);
8436
8437 auto mappable_native_buffer =
8438 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8439 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8440 mappable_native_buffer->GetMappedFramedBuffers();
8441 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8442 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8443 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8444 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8445
8446 if (!allow_i420_conversion_) {
8447 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8448 }
8449}
8450
Erik Språng7444b192021-06-02 14:02:13 +02008451TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8452 if (codec_type_ == kVideoCodecMultiplex) {
8453 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8454 return;
8455 }
8456
8457 const size_t kNumSpatialLayers = 3u;
8458 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8459 const int kFrameWidth = 1280;
8460 const int kFrameHeight = 720;
8461 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8462 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8463 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8464 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8465 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8466 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8467
8468 VideoEncoderConfig config;
8469 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8470 test::FillEncoderConfiguration(codec_type_, 1, &config);
8471 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8472 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8473 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8474 vp9_settings.numberOfTemporalLayers = 3;
8475 vp9_settings.automaticResizeOn = false;
8476 config.encoder_specific_settings =
8477 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8478 vp9_settings);
8479 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8480 /*fps=*/30.0,
8481 /*first_active_layer=*/0,
8482 /*num_spatial_layers=*/3,
8483 /*num_temporal_layers=*/3,
8484 /*is_screenshare=*/false);
8485 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8486 test::FillEncoderConfiguration(codec_type_, 1, &config);
8487 config.max_bitrate_bps = kSimulcastTargetBitrateBps;
8488 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8489 /*fps=*/30.0,
8490 /*first_active_layer=*/0,
8491 /*num_spatial_layers=*/3,
8492 /*num_temporal_layers=*/3,
8493 /*is_screenshare=*/false);
8494 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8495 } else {
8496 // Simulcast for VP8/H264.
8497 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8498 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8499 config.simulcast_layers[i].scale_resolution_down_by =
8500 kDownscaleFactors[i];
8501 config.simulcast_layers[i].active = true;
8502 }
8503 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8504 // Turn off frame dropping to prevent flakiness.
8505 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8506 h264_settings.frameDroppingOn = false;
8507 config.encoder_specific_settings = rtc::make_ref_counted<
8508 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8509 }
8510 }
8511
8512 auto set_layer_active = [&](int layer_idx, bool active) {
8513 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8514 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8515 config.spatial_layers[layer_idx].active = active;
8516 } else {
8517 config.simulcast_layers[layer_idx].active = active;
8518 }
8519 };
8520
8521 config.video_stream_factory =
8522 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8523 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8524 /*screencast*/ false,
8525 /*screenshare enabled*/ false);
8526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8527 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8528 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
8529 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
8530
8531 // Capture a frame with all layers active.
8532 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8533 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8534 int64_t timestamp_ms = kFrameIntervalMs;
8535 video_source_.IncomingCapturedFrame(
8536 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8537
8538 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8539 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8540
8541 // Capture a frame with one of the layers inactive.
8542 set_layer_active(2, false);
8543 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8544 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8545 timestamp_ms += kFrameIntervalMs;
8546 video_source_.IncomingCapturedFrame(
8547 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8548 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8549
8550 // New target bitrates signaled based on lower resolution.
8551 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8553 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8554 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8555
8556 // Re-enable the top layer.
8557 set_layer_active(2, true);
8558 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8559 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8560 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8561
8562 // Bitrate target adjusted back up to enable HD layer...
8563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8564 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8565 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8566 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8567
8568 // ...then add a new frame.
8569 timestamp_ms += kFrameIntervalMs;
8570 video_source_.IncomingCapturedFrame(
8571 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8572 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8573 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8574
8575 video_stream_encoder_->Stop();
8576}
8577
Henrik Boström56db9ff2021-03-24 09:06:45 +01008578std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8579 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8580 VideoCodecType codec_type = std::get<0>(info.param);
8581 bool allow_i420_conversion = std::get<1>(info.param);
8582 std::string str;
8583 switch (codec_type) {
8584 case kVideoCodecGeneric:
8585 str = "Generic";
8586 break;
8587 case kVideoCodecVP8:
8588 str = "VP8";
8589 break;
8590 case kVideoCodecVP9:
8591 str = "VP9";
8592 break;
8593 case kVideoCodecAV1:
8594 str = "AV1";
8595 break;
8596 case kVideoCodecH264:
8597 str = "H264";
8598 break;
8599 case kVideoCodecMultiplex:
8600 str = "Multiplex";
8601 break;
8602 default:
8603 RTC_NOTREACHED();
8604 }
8605 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8606 return str;
8607}
8608
8609constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8610 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8611constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8612 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8613constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8614 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8615constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8616 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8617#if defined(WEBRTC_USE_H264)
8618constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8619 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8620
8621// The windows compiler does not tolerate #if statements inside the
8622// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8623// and without H264).
8624INSTANTIATE_TEST_SUITE_P(
8625 All,
8626 VideoStreamEncoderWithRealEncoderTest,
8627 ::testing::Values(kVP8DisallowConversion,
8628 kVP9DisallowConversion,
8629 kAV1AllowConversion,
8630 kMultiplexDisallowConversion,
8631 kH264AllowConversion),
8632 TestParametersVideoCodecAndAllowI420ConversionToString);
8633#else
8634INSTANTIATE_TEST_SUITE_P(
8635 All,
8636 VideoStreamEncoderWithRealEncoderTest,
8637 ::testing::Values(kVP8DisallowConversion,
8638 kVP9DisallowConversion,
8639 kAV1AllowConversion,
8640 kMultiplexDisallowConversion),
8641 TestParametersVideoCodecAndAllowI420ConversionToString);
8642#endif
8643
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008644class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8645 protected:
8646 void RunTest(const std::vector<VideoStream>& configs,
8647 const int expected_num_init_encode) {
8648 ConfigureEncoder(configs[0]);
8649 OnBitrateUpdated(DataRate::BitsPerSec(kTargetBitrateBps));
8650 InsertFrameAndWaitForEncoded();
8651 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8652 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
8653 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
8654 ExpectEqual(fake_encoder_.video_codec(), configs[0]);
8655
8656 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8657 ConfigureEncoder(configs[1]);
8658 InsertFrameAndWaitForEncoded();
8659 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8660 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
8661 EXPECT_EQ(expected_num_init_encode,
8662 fake_encoder_.GetNumEncoderInitializations());
8663 if (expected_num_init_encode > 1)
8664 ExpectEqual(fake_encoder_.video_codec(), configs[1]);
8665
8666 video_stream_encoder_->Stop();
8667 }
8668
8669 void ConfigureEncoder(const VideoStream& stream) {
8670 VideoEncoderConfig config;
8671 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8672 config.max_bitrate_bps = stream.max_bitrate_bps;
8673 config.simulcast_layers[0] = stream;
8674 config.video_stream_factory =
8675 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8676 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8677 /*conference_mode=*/false);
8678 video_stream_encoder_->ConfigureEncoder(std::move(config),
8679 kMaxPayloadLength);
8680 }
8681
8682 void OnBitrateUpdated(DataRate bitrate) {
8683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8684 bitrate, bitrate, bitrate, 0, 0, 0);
8685 }
8686
8687 void InsertFrameAndWaitForEncoded() {
8688 timestamp_ms_ += kFrameIntervalMs;
8689 video_source_.IncomingCapturedFrame(
8690 CreateFrame(timestamp_ms_, kWidth, kHeight));
8691 sink_.WaitForEncodedFrame(timestamp_ms_);
8692 }
8693
8694 void ExpectEqual(const VideoCodec& actual,
8695 const VideoStream& expected) const {
8696 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8697 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8698 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8699 static_cast<unsigned int>(expected.min_bitrate_bps));
8700 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8701 static_cast<unsigned int>(expected.max_bitrate_bps));
8702 EXPECT_EQ(actual.simulcastStream[0].width,
8703 kWidth / expected.scale_resolution_down_by);
8704 EXPECT_EQ(actual.simulcastStream[0].height,
8705 kHeight / expected.scale_resolution_down_by);
8706 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8707 expected.num_temporal_layers);
8708 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8709 }
8710
8711 VideoStream DefaultConfig() const {
8712 VideoStream stream;
8713 stream.max_framerate = 25;
8714 stream.min_bitrate_bps = 35000;
8715 stream.max_bitrate_bps = 900000;
8716 stream.scale_resolution_down_by = 1.0;
8717 stream.num_temporal_layers = 1;
8718 stream.bitrate_priority = 1.0;
8719 stream.scalability_mode = "";
8720 return stream;
8721 }
8722
8723 const int kWidth = 640;
8724 const int kHeight = 360;
8725 int64_t timestamp_ms_ = 0;
8726};
8727
8728TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8729 VideoStream config1 = DefaultConfig();
8730 VideoStream config2 = config1;
8731 config2.max_framerate++;
8732
8733 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8734}
8735
8736TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8737 VideoStream config1 = DefaultConfig();
8738 VideoStream config2 = config1;
8739 config2.min_bitrate_bps += 10000;
8740
8741 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8742}
8743
8744TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8745 VideoStream config1 = DefaultConfig();
8746 VideoStream config2 = config1;
8747 config2.max_bitrate_bps += 100000;
8748
8749 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8750}
8751
8752TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8753 VideoStream config1 = DefaultConfig();
8754 VideoStream config2 = config1;
8755 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8756
8757 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8758}
8759
8760TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8761 VideoStream config1 = DefaultConfig();
8762 VideoStream config2 = config1;
8763 config2.scale_resolution_down_by *= 2;
8764
8765 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8766}
8767
8768TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8769 VideoStream config1 = DefaultConfig();
8770 VideoStream config2 = config1;
8771 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8772
8773 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8774}
8775
8776TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8777 VideoStream config1 = DefaultConfig();
8778 VideoStream config2 = config1;
8779 config2.scalability_mode = "L1T2";
8780
8781 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8782}
8783
perkj26091b12016-09-01 01:17:40 -07008784} // namespace webrtc