blob: fcfa6778aea273b1b0c1a5f4c8ec6040141f6b38 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020045#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010046#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020047#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010048#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020049#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080051#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020052#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010053#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020054#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "test/encoder_settings.h"
56#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020057#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010058#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/gmock.h"
60#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010061#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020062#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020063#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020064#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070065
66namespace webrtc {
67
sprang57c2fff2017-01-16 06:24:02 -080068using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020069using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020070using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020071using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020073using ::testing::Ge;
74using ::testing::Gt;
75using ::testing::Le;
76using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010077using ::testing::Matcher;
78using ::testing::NiceMock;
79using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010080using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020081using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080082
perkj803d97f2016-11-01 11:45:46 -070083namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020084const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010085const int kQpLow = 1;
86const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int kMinFramerateFps = 2;
88const int kMinBalancedFramerateFps = 7;
89const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080090const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010091const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020092const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010093const uint32_t kSimulcastTargetBitrateBps = 3150000;
94const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080095const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070096const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020097const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020098const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020099const VideoEncoder::ResolutionBitrateLimits
100 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
101const VideoEncoder::ResolutionBitrateLimits
102 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800103
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200104uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
105 0x00, 0x00, 0x03, 0x03, 0xF4,
106 0x05, 0x03, 0xC7, 0xE0, 0x1B,
107 0x41, 0x10, 0x8D, 0x00};
108
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100109const uint8_t kCodedFrameVp8Qp25[] = {
110 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
111 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
112 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
113
perkj803d97f2016-11-01 11:45:46 -0700114class TestBuffer : public webrtc::I420Buffer {
115 public:
116 TestBuffer(rtc::Event* event, int width, int height)
117 : I420Buffer(width, height), event_(event) {}
118
119 private:
120 friend class rtc::RefCountedObject<TestBuffer>;
121 ~TestBuffer() override {
122 if (event_)
123 event_->Set();
124 }
125 rtc::Event* const event_;
126};
127
Henrik Boström56db9ff2021-03-24 09:06:45 +0100128// A fake native buffer that can't be converted to I420. Upon scaling, it
129// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700130class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
131 public:
132 FakeNativeBuffer(rtc::Event* event, int width, int height)
133 : event_(event), width_(width), height_(height) {}
134 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
135 int width() const override { return width_; }
136 int height() const override { return height_; }
137 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
138 return nullptr;
139 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100140 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
141 int offset_x,
142 int offset_y,
143 int crop_width,
144 int crop_height,
145 int scaled_width,
146 int scaled_height) override {
147 return new rtc::RefCountedObject<FakeNativeBuffer>(nullptr, scaled_width,
148 scaled_height);
149 }
Noah Richards51db4212019-06-12 06:59:12 -0700150
151 private:
152 friend class rtc::RefCountedObject<FakeNativeBuffer>;
153 ~FakeNativeBuffer() override {
154 if (event_)
155 event_->Set();
156 }
157 rtc::Event* const event_;
158 const int width_;
159 const int height_;
160};
161
Evan Shrubsole895556e2020-10-05 09:15:13 +0200162// A fake native buffer that is backed by an NV12 buffer.
163class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
164 public:
165 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
166 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
167
168 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
169 int width() const override { return nv12_buffer_->width(); }
170 int height() const override { return nv12_buffer_->height(); }
171 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
172 return nv12_buffer_->ToI420();
173 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200174 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
175 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
176 if (absl::c_find(types, Type::kNV12) != types.end()) {
177 return nv12_buffer_;
178 }
179 return nullptr;
180 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200181 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
182
183 private:
184 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
185 ~FakeNV12NativeBuffer() override {
186 if (event_)
187 event_->Set();
188 }
189 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
190 rtc::Event* const event_;
191};
192
Niels Möller7dc26b72017-12-06 10:27:48 +0100193class CpuOveruseDetectorProxy : public OveruseFrameDetector {
194 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200195 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
196 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200197 last_target_framerate_fps_(-1),
198 framerate_updated_event_(true /* manual_reset */,
199 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100200 virtual ~CpuOveruseDetectorProxy() {}
201
202 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200203 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100204 last_target_framerate_fps_ = framerate_fps;
205 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200206 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100207 }
208
209 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200210 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100211 return last_target_framerate_fps_;
212 }
213
Niels Möller4db138e2018-04-19 09:04:13 +0200214 CpuOveruseOptions GetOptions() { return options_; }
215
Henrik Boström381d1092020-05-12 18:49:07 +0200216 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
217
Niels Möller7dc26b72017-12-06 10:27:48 +0100218 private:
Markus Handella3765182020-07-08 13:13:32 +0200219 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100220 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200221 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100222};
223
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200224class FakeVideoSourceRestrictionsListener
225 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200226 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200227 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200228 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200229 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200230 RTC_DCHECK(was_restrictions_updated_);
231 }
232
233 rtc::Event* restrictions_updated_event() {
234 return &restrictions_updated_event_;
235 }
236
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200237 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200238 void OnVideoSourceRestrictionsUpdated(
239 VideoSourceRestrictions restrictions,
240 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200241 rtc::scoped_refptr<Resource> reason,
242 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200243 was_restrictions_updated_ = true;
244 restrictions_updated_event_.Set();
245 }
246
247 private:
248 bool was_restrictions_updated_;
249 rtc::Event restrictions_updated_event_;
250};
251
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252auto WantsFps(Matcher<int> fps_matcher) {
253 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
254 fps_matcher);
255}
256
257auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
258 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
259 AllOf(max_pixel_matcher, Gt(0)));
260}
261
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200262auto ResolutionMax() {
263 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200264 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200265 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
266 Eq(absl::nullopt)));
267}
268
269auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200270 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200271}
272
273auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200274 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200275}
276
277auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200278 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200279}
280
281auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200282 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200283}
284
285auto FpsMaxResolutionMax() {
286 return AllOf(FpsMax(), ResolutionMax());
287}
288
289auto UnlimitedSinkWants() {
290 return AllOf(FpsUnlimited(), ResolutionMax());
291}
292
293auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
294 Matcher<int> fps_range_matcher;
295
296 if (last_frame_pixels <= 320 * 240) {
297 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200298 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200299 fps_range_matcher = AllOf(Ge(10), Le(15));
300 } else if (last_frame_pixels <= 640 * 480) {
301 fps_range_matcher = Ge(15);
302 } else {
303 fps_range_matcher = Eq(kDefaultFramerate);
304 }
305 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
306 fps_range_matcher);
307}
308
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
310 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
311 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
312}
313
314auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
315 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
316}
317
318auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
319 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
320}
321
322auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
323 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
324 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
325}
326
327auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
328 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
329 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
330}
331
332auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
333 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
334 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
335}
336
337auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
338 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
339 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
340}
341
mflodmancc3d4422017-08-03 08:27:51 -0700342class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700343 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200344 VideoStreamEncoderUnderTest(TimeController* time_controller,
345 TaskQueueFactory* task_queue_factory,
346 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100347 const VideoStreamEncoderSettings& settings,
348 VideoStreamEncoder::BitrateAllocationCallbackType
349 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200350 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100351 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200352 stats_proxy,
353 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200354 std::unique_ptr<OveruseFrameDetector>(
355 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100356 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100357 task_queue_factory,
358 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200359 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200360 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200361 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200362 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200363 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200364 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200365 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200366 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100367 }
perkj803d97f2016-11-01 11:45:46 -0700368
Henrik Boström381d1092020-05-12 18:49:07 +0200369 void SetSourceAndWaitForRestrictionsUpdated(
370 rtc::VideoSourceInterface<VideoFrame>* source,
371 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200372 FakeVideoSourceRestrictionsListener listener;
373 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200374 SetSource(source, degradation_preference);
375 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200376 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200377 }
378
379 void SetSourceAndWaitForFramerateUpdated(
380 rtc::VideoSourceInterface<VideoFrame>* source,
381 const DegradationPreference& degradation_preference) {
382 overuse_detector_proxy_->framerate_updated_event()->Reset();
383 SetSource(source, degradation_preference);
384 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
385 }
386
387 void OnBitrateUpdatedAndWaitForManagedResources(
388 DataRate target_bitrate,
389 DataRate stable_target_bitrate,
390 DataRate link_allocation,
391 uint8_t fraction_lost,
392 int64_t round_trip_time_ms,
393 double cwnd_reduce_ratio) {
394 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
395 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
396 // Bitrate is updated on the encoder queue.
397 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200398 }
399
kthelgason2fc52542017-03-03 00:24:41 -0800400 // This is used as a synchronisation mechanism, to make sure that the
401 // encoder queue is not blocked before we start sending it frames.
402 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100403 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200404 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800405 ASSERT_TRUE(event.Wait(5000));
406 }
407
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200409 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200410 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200411 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200412 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200413 event.Set();
414 });
415 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200416 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200418
Henrik Boström91aa7322020-04-28 12:24:33 +0200419 void TriggerCpuUnderuse() {
420 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200421 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200422 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200423 event.Set();
424 });
425 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200426 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200427 }
kthelgason876222f2016-11-29 01:44:11 -0800428
Henrik Boström91aa7322020-04-28 12:24:33 +0200429 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200430 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200431 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200432 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200433 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200434 event.Set();
435 });
436 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200437 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200438 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200441 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200442 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 event.Set();
444 });
445 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200446 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 }
448
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200449 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100450 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200451 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
452 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200453 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700454};
455
Noah Richards51db4212019-06-12 06:59:12 -0700456// Simulates simulcast behavior and makes highest stream resolutions divisible
457// by 4.
458class CroppingVideoStreamFactory
459 : public VideoEncoderConfig::VideoStreamFactoryInterface {
460 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200461 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700462
463 private:
464 std::vector<VideoStream> CreateEncoderStreams(
465 int width,
466 int height,
467 const VideoEncoderConfig& encoder_config) override {
468 std::vector<VideoStream> streams = test::CreateVideoStreams(
469 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700470 return streams;
471 }
Noah Richards51db4212019-06-12 06:59:12 -0700472};
473
sprangb1ca0732017-02-01 08:38:12 -0800474class AdaptingFrameForwarder : public test::FrameForwarder {
475 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200476 explicit AdaptingFrameForwarder(TimeController* time_controller)
477 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700478 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800479
480 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200481 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800482 adaptation_enabled_ = enabled;
483 }
484
asaperssonfab67072017-04-04 05:51:49 -0700485 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200486 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800487 return adaptation_enabled_;
488 }
489
Henrik Boström1124ed12021-02-25 10:30:39 +0100490 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
491 // the resolution or frame rate was different than it is currently. If
492 // something else is modified, such as encoder resolutions, but the resolution
493 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700494 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200495 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700496 return last_wants_;
497 }
498
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200499 absl::optional<int> last_sent_width() const { return last_width_; }
500 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800501
sprangb1ca0732017-02-01 08:38:12 -0800502 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200503 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
504 time_controller_->AdvanceTime(TimeDelta::Millis(0));
505
sprangb1ca0732017-02-01 08:38:12 -0800506 int cropped_width = 0;
507 int cropped_height = 0;
508 int out_width = 0;
509 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700510 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
512 << "w=" << video_frame.width()
513 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700514 if (adapter_.AdaptFrameResolution(
515 video_frame.width(), video_frame.height(),
516 video_frame.timestamp_us() * 1000, &cropped_width,
517 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100518 VideoFrame adapted_frame =
519 VideoFrame::Builder()
520 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
521 nullptr, out_width, out_height))
522 .set_timestamp_rtp(99)
523 .set_timestamp_ms(99)
524 .set_rotation(kVideoRotation_0)
525 .build();
sprangc5d62e22017-04-02 23:53:04 -0700526 adapted_frame.set_ntp_time_ms(video_frame.ntp_time_ms());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100527 if (video_frame.has_update_rect()) {
528 adapted_frame.set_update_rect(
529 video_frame.update_rect().ScaleWithFrame(
530 video_frame.width(), video_frame.height(), 0, 0,
531 video_frame.width(), video_frame.height(), out_width,
532 out_height));
533 }
sprangc5d62e22017-04-02 23:53:04 -0700534 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800535 last_width_.emplace(adapted_frame.width());
536 last_height_.emplace(adapted_frame.height());
537 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 last_width_ = absl::nullopt;
539 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700540 }
sprangb1ca0732017-02-01 08:38:12 -0800541 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800543 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800544 last_width_.emplace(video_frame.width());
545 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800546 }
547 }
548
549 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
550 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200551 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100552 rtc::VideoSinkWants prev_wants = sink_wants_locked();
553 bool did_adapt =
554 prev_wants.max_pixel_count != wants.max_pixel_count ||
555 prev_wants.target_pixel_count != wants.target_pixel_count ||
556 prev_wants.max_framerate_fps != wants.max_framerate_fps;
557 if (did_adapt) {
558 last_wants_ = prev_wants;
559 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100560 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200561 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800562 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200563
564 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800565 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200566 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
567 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200568 absl::optional<int> last_width_;
569 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800570};
sprangc5d62e22017-04-02 23:53:04 -0700571
Niels Möller213618e2018-07-24 09:29:58 +0200572// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700573class MockableSendStatisticsProxy : public SendStatisticsProxy {
574 public:
575 MockableSendStatisticsProxy(Clock* clock,
576 const VideoSendStream::Config& config,
577 VideoEncoderConfig::ContentType content_type)
578 : SendStatisticsProxy(clock, config, content_type) {}
579
580 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200581 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700582 if (mock_stats_)
583 return *mock_stats_;
584 return SendStatisticsProxy::GetStats();
585 }
586
Niels Möller213618e2018-07-24 09:29:58 +0200587 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200588 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200589 if (mock_stats_)
590 return mock_stats_->input_frame_rate;
591 return SendStatisticsProxy::GetInputFrameRate();
592 }
sprangc5d62e22017-04-02 23:53:04 -0700593 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200594 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700595 mock_stats_.emplace(stats);
596 }
597
598 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200599 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700600 mock_stats_.reset();
601 }
602
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200603 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
604 on_frame_dropped_ = std::move(callback);
605 }
606
sprangc5d62e22017-04-02 23:53:04 -0700607 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200608 void OnFrameDropped(DropReason reason) override {
609 SendStatisticsProxy::OnFrameDropped(reason);
610 if (on_frame_dropped_)
611 on_frame_dropped_(reason);
612 }
613
Markus Handella3765182020-07-08 13:13:32 +0200614 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200615 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200616 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700617};
618
philipel9b058032020-02-10 11:30:00 +0100619class MockEncoderSelector
620 : public VideoEncoderFactory::EncoderSelectorInterface {
621 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200622 MOCK_METHOD(void,
623 OnCurrentEncoder,
624 (const SdpVideoFormat& format),
625 (override));
626 MOCK_METHOD(absl::optional<SdpVideoFormat>,
627 OnAvailableBitrate,
628 (const DataRate& rate),
629 (override));
630 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100631};
632
perkj803d97f2016-11-01 11:45:46 -0700633} // namespace
634
mflodmancc3d4422017-08-03 08:27:51 -0700635class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700636 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200637 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700638
mflodmancc3d4422017-08-03 08:27:51 -0700639 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700640 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700641 codec_width_(320),
642 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200643 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200644 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200645 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700646 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200647 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700648 video_send_config_,
649 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700651
652 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700653 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700654 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200655 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800656 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200657 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200658 video_send_config_.rtp.payload_name = "FAKE";
659 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700660
Per512ecb32016-09-23 15:52:06 +0200661 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200662 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200663 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
664 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
665 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100666 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700667
Niels Möllerf1338562018-04-26 09:51:47 +0200668 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800669 }
670
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100671 void ConfigureEncoder(
672 VideoEncoderConfig video_encoder_config,
673 VideoStreamEncoder::BitrateAllocationCallbackType
674 allocation_callback_type =
675 VideoStreamEncoder::BitrateAllocationCallbackType::
676 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700677 if (video_stream_encoder_)
678 video_stream_encoder_->Stop();
679 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200680 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100681 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700682 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
683 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700684 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700685 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
686 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200687 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700688 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800689 }
690
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100691 void ResetEncoder(const std::string& payload_name,
692 size_t num_streams,
693 size_t num_temporal_layers,
694 unsigned char num_spatial_layers,
695 bool screenshare,
696 VideoStreamEncoder::BitrateAllocationCallbackType
697 allocation_callback_type =
698 VideoStreamEncoder::BitrateAllocationCallbackType::
699 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200700 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800701
702 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200703 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
704 num_streams, &video_encoder_config);
705 for (auto& layer : video_encoder_config.simulcast_layers) {
706 layer.num_temporal_layers = num_temporal_layers;
707 layer.max_framerate = kDefaultFramerate;
708 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100709 video_encoder_config.max_bitrate_bps =
710 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700711 video_encoder_config.content_type =
712 screenshare ? VideoEncoderConfig::ContentType::kScreen
713 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700714 if (payload_name == "VP9") {
715 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
716 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200717 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700718 video_encoder_config.encoder_specific_settings =
719 new rtc::RefCountedObject<
720 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
721 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100722 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700723 }
724
sprang57c2fff2017-01-16 06:24:02 -0800725 VideoFrame CreateFrame(int64_t ntp_time_ms,
726 rtc::Event* destruction_event) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100727 VideoFrame frame =
728 VideoFrame::Builder()
729 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
730 destruction_event, codec_width_, codec_height_))
731 .set_timestamp_rtp(99)
732 .set_timestamp_ms(99)
733 .set_rotation(kVideoRotation_0)
734 .build();
sprang57c2fff2017-01-16 06:24:02 -0800735 frame.set_ntp_time_ms(ntp_time_ms);
perkj26091b12016-09-01 01:17:40 -0700736 return frame;
737 }
738
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100739 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
740 rtc::Event* destruction_event,
741 int offset_x) const {
742 VideoFrame frame =
743 VideoFrame::Builder()
744 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
745 destruction_event, codec_width_, codec_height_))
746 .set_timestamp_rtp(99)
747 .set_timestamp_ms(99)
748 .set_rotation(kVideoRotation_0)
Artem Titov5256d8b2019-12-02 10:34:12 +0100749 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100750 .build();
751 frame.set_ntp_time_ms(ntp_time_ms);
752 return frame;
753 }
754
sprang57c2fff2017-01-16 06:24:02 -0800755 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100756 VideoFrame frame =
757 VideoFrame::Builder()
758 .set_video_frame_buffer(
759 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
760 .set_timestamp_rtp(99)
761 .set_timestamp_ms(99)
762 .set_rotation(kVideoRotation_0)
763 .build();
sprang57c2fff2017-01-16 06:24:02 -0800764 frame.set_ntp_time_ms(ntp_time_ms);
sprangc5d62e22017-04-02 23:53:04 -0700765 frame.set_timestamp_us(ntp_time_ms * 1000);
perkj803d97f2016-11-01 11:45:46 -0700766 return frame;
767 }
768
Evan Shrubsole895556e2020-10-05 09:15:13 +0200769 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
770 VideoFrame frame =
771 VideoFrame::Builder()
772 .set_video_frame_buffer(NV12Buffer::Create(width, height))
773 .set_timestamp_rtp(99)
774 .set_timestamp_ms(99)
775 .set_rotation(kVideoRotation_0)
776 .build();
777 frame.set_ntp_time_ms(ntp_time_ms);
778 frame.set_timestamp_us(ntp_time_ms * 1000);
779 return frame;
780 }
781
Noah Richards51db4212019-06-12 06:59:12 -0700782 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
783 rtc::Event* destruction_event,
784 int width,
785 int height) const {
786 VideoFrame frame =
787 VideoFrame::Builder()
788 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
789 destruction_event, width, height))
790 .set_timestamp_rtp(99)
791 .set_timestamp_ms(99)
792 .set_rotation(kVideoRotation_0)
793 .build();
794 frame.set_ntp_time_ms(ntp_time_ms);
795 return frame;
796 }
797
Evan Shrubsole895556e2020-10-05 09:15:13 +0200798 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
799 rtc::Event* destruction_event,
800 int width,
801 int height) const {
802 VideoFrame frame = VideoFrame::Builder()
803 .set_video_frame_buffer(
804 new rtc::RefCountedObject<FakeNV12NativeBuffer>(
805 destruction_event, width, height))
806 .set_timestamp_rtp(99)
807 .set_timestamp_ms(99)
808 .set_rotation(kVideoRotation_0)
809 .build();
810 frame.set_ntp_time_ms(ntp_time_ms);
811 return frame;
812 }
813
Noah Richards51db4212019-06-12 06:59:12 -0700814 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
815 rtc::Event* destruction_event) const {
816 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
817 codec_height_);
818 }
819
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100820 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100822 DataRate::BitsPerSec(kTargetBitrateBps),
823 DataRate::BitsPerSec(kTargetBitrateBps),
824 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100825
826 video_source_.IncomingCapturedFrame(
827 CreateFrame(1, codec_width_, codec_height_));
828 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200829 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100830 }
831
sprang4847ae62017-06-27 07:06:52 -0700832 void WaitForEncodedFrame(int64_t expected_ntp_time) {
833 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700835 }
836
837 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
838 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200839 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700840 return ok;
841 }
842
843 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
844 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200845 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700846 }
847
848 void ExpectDroppedFrame() {
849 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200850 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700851 }
852
853 bool WaitForFrame(int64_t timeout_ms) {
854 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200855 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700856 return ok;
857 }
858
perkj26091b12016-09-01 01:17:40 -0700859 class TestEncoder : public test::FakeEncoder {
860 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200861 explicit TestEncoder(TimeController* time_controller)
862 : FakeEncoder(time_controller->GetClock()),
863 time_controller_(time_controller) {
864 RTC_DCHECK(time_controller_);
865 }
perkj26091b12016-09-01 01:17:40 -0700866
asaperssonfab67072017-04-04 05:51:49 -0700867 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200868 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700869 return config_;
870 }
871
872 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200873 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700874 block_next_encode_ = true;
875 }
876
Erik Språngaed30702018-11-05 12:57:17 +0100877 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200878 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200879 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100880 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100881 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100882 info.scaling_settings = VideoEncoder::ScalingSettings(
883 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100884 }
885 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100886 for (int i = 0; i < kMaxSpatialLayers; ++i) {
887 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100888 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100889 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100890 for (int tid = 0; tid < num_layers; ++tid)
891 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100892 }
893 }
Erik Språngaed30702018-11-05 12:57:17 +0100894 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200895
896 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100897 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200898 info.apply_alignment_to_all_simulcast_layers =
899 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200900 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100901 return info;
kthelgason876222f2016-11-29 01:44:11 -0800902 }
903
Erik Språngb7cb7b52019-02-26 15:52:33 +0100904 int32_t RegisterEncodeCompleteCallback(
905 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200906 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100907 encoded_image_callback_ = callback;
908 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
909 }
910
perkjfa10b552016-10-02 23:45:26 -0700911 void ContinueEncode() { continue_encode_event_.Set(); }
912
913 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
914 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200915 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700916 EXPECT_EQ(timestamp_, timestamp);
917 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
918 }
919
kthelgason2fc52542017-03-03 00:24:41 -0800920 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800922 quality_scaling_ = b;
923 }
kthelgasonad9010c2017-02-14 00:46:51 -0800924
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100925 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200926 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100927 requested_resolution_alignment_ = requested_resolution_alignment;
928 }
929
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200930 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
931 MutexLock lock(&local_mutex_);
932 apply_alignment_to_all_simulcast_layers_ = b;
933 }
934
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100935 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200936 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100937 is_hardware_accelerated_ = is_hardware_accelerated;
938 }
939
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100940 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
941 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200942 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100943 temporal_layers_supported_[spatial_idx] = supported;
944 }
945
Sergey Silkin6456e352019-07-08 17:56:40 +0200946 void SetResolutionBitrateLimits(
947 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200948 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200949 resolution_bitrate_limits_ = thresholds;
950 }
951
sprangfe627f32017-03-29 08:24:59 -0700952 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200953 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700954 force_init_encode_failed_ = force_failure;
955 }
956
Niels Möller6bb5ab92019-01-11 11:11:10 +0100957 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200958 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100959 rate_factor_ = rate_factor;
960 }
961
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200963 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100964 return last_framerate_;
965 }
966
Erik Språngd7329ca2019-02-21 21:19:53 +0100967 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200968 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100969 return last_update_rect_;
970 }
971
Niels Möller87e2d782019-03-07 10:18:23 +0100972 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200973 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100974 return last_frame_types_;
975 }
976
977 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100978 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100979 keyframe ? VideoFrameType::kVideoFrameKey
980 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100981 {
Markus Handella3765182020-07-08 13:13:32 +0200982 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100983 last_frame_types_ = frame_type;
984 }
Niels Möllerb859b322019-03-07 12:40:01 +0100985 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100986 }
987
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100988 void InjectEncodedImage(const EncodedImage& image,
989 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200990 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100991 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100992 }
993
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200994 void SetEncodedImageData(
995 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200996 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200997 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200998 }
999
Erik Språngd7329ca2019-02-21 21:19:53 +01001000 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001001 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001002 expect_null_frame_ = true;
1003 }
1004
Erik Språng5056af02019-09-02 15:53:11 +02001005 absl::optional<VideoEncoder::RateControlParameters>
1006 GetAndResetLastRateControlSettings() {
1007 auto settings = last_rate_control_settings_;
1008 last_rate_control_settings_.reset();
1009 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001010 }
1011
Henrik Boström56db9ff2021-03-24 09:06:45 +01001012 int GetLastInputWidth() const {
1013 MutexLock lock(&local_mutex_);
1014 return last_input_width_;
1015 }
1016
1017 int GetLastInputHeight() const {
1018 MutexLock lock(&local_mutex_);
1019 return last_input_height_;
1020 }
1021
Evan Shrubsole895556e2020-10-05 09:15:13 +02001022 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1023 MutexLock lock(&local_mutex_);
1024 return last_input_pixel_format_;
1025 }
1026
Sergey Silkin5ee69672019-07-02 14:18:34 +02001027 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +02001028 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001029 return num_encoder_initializations_;
1030 }
1031
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001032 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001033 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001034 return num_set_rates_;
1035 }
1036
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001037 VideoCodec video_codec() const {
1038 MutexLock lock(&local_mutex_);
1039 return video_codec_;
1040 }
1041
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001042 void SetPreferredPixelFormats(
1043 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1044 pixel_formats) {
1045 MutexLock lock(&local_mutex_);
1046 preferred_pixel_formats_ = std::move(pixel_formats);
1047 }
1048
perkjfa10b552016-10-02 23:45:26 -07001049 private:
perkj26091b12016-09-01 01:17:40 -07001050 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001051 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001052 bool block_encode;
1053 {
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001055 if (expect_null_frame_) {
1056 EXPECT_EQ(input_image.timestamp(), 0u);
1057 EXPECT_EQ(input_image.width(), 1);
1058 last_frame_types_ = *frame_types;
1059 expect_null_frame_ = false;
1060 } else {
1061 EXPECT_GT(input_image.timestamp(), timestamp_);
1062 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1063 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1064 }
perkj26091b12016-09-01 01:17:40 -07001065
1066 timestamp_ = input_image.timestamp();
1067 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001068 last_input_width_ = input_image.width();
1069 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001070 block_encode = block_next_encode_;
1071 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001072 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001073 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001074 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001075 }
Niels Möllerb859b322019-03-07 12:40:01 +01001076 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001077 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001078 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001079
perkj26091b12016-09-01 01:17:40 -07001080 return result;
1081 }
1082
Niels Möller08ae7ce2020-09-23 15:58:12 +02001083 CodecSpecificInfo EncodeHook(
1084 EncodedImage& encoded_image,
1085 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001086 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001087 {
1088 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001089 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001090 }
1091 MutexLock lock(&local_mutex_);
1092 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001093 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001094 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001095 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001096 }
1097
sprangfe627f32017-03-29 08:24:59 -07001098 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001099 const Settings& settings) override {
1100 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001101
Markus Handella3765182020-07-08 13:13:32 +02001102 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001103 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001104
1105 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001106 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001107
Erik Språng82fad3d2018-03-21 09:57:23 +01001108 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001109 // Simulate setting up temporal layers, in order to validate the life
1110 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001111 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001112 frame_buffer_controller_ =
1113 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001114 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001115 if (force_init_encode_failed_) {
1116 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001117 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001118 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001119
Erik Språngb7cb7b52019-02-26 15:52:33 +01001120 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001121 return res;
1122 }
1123
Erik Språngb7cb7b52019-02-26 15:52:33 +01001124 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001125 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001126 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1127 initialized_ = EncoderState::kUninitialized;
1128 return FakeEncoder::Release();
1129 }
1130
Erik Språng16cb8f52019-04-12 13:59:09 +02001131 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001133 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001134 VideoBitrateAllocation adjusted_rate_allocation;
1135 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1136 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001137 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001138 adjusted_rate_allocation.SetBitrate(
1139 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001140 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001141 rate_factor_));
1142 }
1143 }
1144 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001145 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001146 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001147 RateControlParameters adjusted_paramters = parameters;
1148 adjusted_paramters.bitrate = adjusted_rate_allocation;
1149 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001150 }
1151
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001152 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001153 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001154 enum class EncoderState {
1155 kUninitialized,
1156 kInitializationFailed,
1157 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001158 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1159 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001160 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001161 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1162 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1163 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1164 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1165 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1166 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001167 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1168 false;
Markus Handella3765182020-07-08 13:13:32 +02001169 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001170 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1171 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001172 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001173 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001174 absl::optional<bool>
1175 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001176 local_mutex_);
1177 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1178 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1179 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001180 absl::optional<VideoEncoder::RateControlParameters>
1181 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001182 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1183 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001184 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001185 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001186 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1187 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001188 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001189 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001190 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001191 RTC_GUARDED_BY(local_mutex_);
1192 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001193 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001194 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1195 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001196 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1197 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001198 };
1199
mflodmancc3d4422017-08-03 08:27:51 -07001200 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001201 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001202 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1203 : time_controller_(time_controller), test_encoder_(test_encoder) {
1204 RTC_DCHECK(time_controller_);
1205 }
perkj26091b12016-09-01 01:17:40 -07001206
perkj26091b12016-09-01 01:17:40 -07001207 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001208 EXPECT_TRUE(
1209 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1210 }
1211
1212 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1213 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001214 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001215 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001216 return false;
perkj26091b12016-09-01 01:17:40 -07001217 {
Markus Handella3765182020-07-08 13:13:32 +02001218 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001219 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001220 }
1221 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001222 return true;
perkj26091b12016-09-01 01:17:40 -07001223 }
1224
sprangb1ca0732017-02-01 08:38:12 -08001225 void WaitForEncodedFrame(uint32_t expected_width,
1226 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001227 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001228 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001229 }
1230
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001231 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001232 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001233 uint32_t width = 0;
1234 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001235 {
Markus Handella3765182020-07-08 13:13:32 +02001236 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001237 width = last_width_;
1238 height = last_height_;
1239 }
1240 EXPECT_EQ(expected_height, height);
1241 EXPECT_EQ(expected_width, width);
1242 }
1243
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001244 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1245 VideoRotation rotation;
1246 {
Markus Handella3765182020-07-08 13:13:32 +02001247 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001248 rotation = last_rotation_;
1249 }
1250 EXPECT_EQ(expected_rotation, rotation);
1251 }
1252
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001253 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001254
sprangc5d62e22017-04-02 23:53:04 -07001255 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001256 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1257 bool ret = encoded_frame_event_.Wait(timeout_ms);
1258 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1259 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001260 }
1261
perkj26091b12016-09-01 01:17:40 -07001262 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001263 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001264 expect_frames_ = false;
1265 }
1266
asaperssonfab67072017-04-04 05:51:49 -07001267 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001269 return number_of_reconfigurations_;
1270 }
1271
asaperssonfab67072017-04-04 05:51:49 -07001272 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001273 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001274 return min_transmit_bitrate_bps_;
1275 }
1276
Erik Språngd7329ca2019-02-21 21:19:53 +01001277 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001278 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001279 num_expected_layers_ = num_layers;
1280 }
1281
Erik Språngb7cb7b52019-02-26 15:52:33 +01001282 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001283 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001284 return last_capture_time_ms_;
1285 }
1286
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001287 const EncodedImage& GetLastEncodedImage() {
1288 MutexLock lock(&mutex_);
1289 return last_encoded_image_;
1290 }
1291
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001292 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001293 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001294 return std::move(last_encoded_image_data_);
1295 }
1296
Per Kjellanderdcef6412020-10-07 15:09:05 +02001297 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1298 MutexLock lock(&mutex_);
1299 return last_bitrate_allocation_;
1300 }
1301
1302 int number_of_bitrate_allocations() const {
1303 MutexLock lock(&mutex_);
1304 return number_of_bitrate_allocations_;
1305 }
1306
Per Kjellandera9434842020-10-15 17:53:22 +02001307 VideoLayersAllocation GetLastVideoLayersAllocation() {
1308 MutexLock lock(&mutex_);
1309 return last_layers_allocation_;
1310 }
1311
1312 int number_of_layers_allocations() const {
1313 MutexLock lock(&mutex_);
1314 return number_of_layers_allocations_;
1315 }
1316
perkj26091b12016-09-01 01:17:40 -07001317 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001318 Result OnEncodedImage(
1319 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001320 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001321 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001322 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001323 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001324 last_encoded_image_data_ = std::vector<uint8_t>(
1325 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001326 uint32_t timestamp = encoded_image.Timestamp();
1327 if (last_timestamp_ != timestamp) {
1328 num_received_layers_ = 1;
1329 } else {
1330 ++num_received_layers_;
1331 }
1332 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001333 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001334 last_width_ = encoded_image._encodedWidth;
1335 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001336 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001337 if (num_received_layers_ == num_expected_layers_) {
1338 encoded_frame_event_.Set();
1339 }
sprangb1ca0732017-02-01 08:38:12 -08001340 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001341 }
1342
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001343 void OnEncoderConfigurationChanged(
1344 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001345 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001346 VideoEncoderConfig::ContentType content_type,
1347 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001348 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001349 ++number_of_reconfigurations_;
1350 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1351 }
1352
Per Kjellanderdcef6412020-10-07 15:09:05 +02001353 void OnBitrateAllocationUpdated(
1354 const VideoBitrateAllocation& allocation) override {
1355 MutexLock lock(&mutex_);
1356 ++number_of_bitrate_allocations_;
1357 last_bitrate_allocation_ = allocation;
1358 }
1359
Per Kjellandera9434842020-10-15 17:53:22 +02001360 void OnVideoLayersAllocationUpdated(
1361 VideoLayersAllocation allocation) override {
1362 MutexLock lock(&mutex_);
1363 ++number_of_layers_allocations_;
1364 last_layers_allocation_ = allocation;
1365 rtc::StringBuilder log;
1366 for (const auto& layer : allocation.active_spatial_layers) {
1367 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1368 << "[";
1369 for (const auto target_bitrate :
1370 layer.target_bitrate_per_temporal_layer) {
1371 log << target_bitrate.kbps() << ",";
1372 }
1373 log << "]";
1374 }
1375 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1376 }
1377
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001378 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001379 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001380 TestEncoder* test_encoder_;
1381 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001382 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001383 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001384 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001385 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001386 uint32_t last_height_ = 0;
1387 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001388 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001389 size_t num_expected_layers_ = 1;
1390 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001391 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001392 int number_of_reconfigurations_ = 0;
1393 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001394 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1395 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001396 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1397 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001398 };
1399
Sergey Silkin5ee69672019-07-02 14:18:34 +02001400 class VideoBitrateAllocatorProxyFactory
1401 : public VideoBitrateAllocatorFactory {
1402 public:
1403 VideoBitrateAllocatorProxyFactory()
1404 : bitrate_allocator_factory_(
1405 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1406
1407 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1408 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001409 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001410 codec_config_ = codec;
1411 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1412 }
1413
1414 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001415 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001416 return codec_config_;
1417 }
1418
1419 private:
1420 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1421
Markus Handella3765182020-07-08 13:13:32 +02001422 mutable Mutex mutex_;
1423 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001424 };
1425
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001426 Clock* clock() { return time_controller_.GetClock(); }
1427 void AdvanceTime(TimeDelta duration) {
1428 time_controller_.AdvanceTime(duration);
1429 }
1430
1431 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1432
1433 protected:
1434 virtual TaskQueueFactory* GetTaskQueueFactory() {
1435 return time_controller_.GetTaskQueueFactory();
1436 }
1437
1438 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001439 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001440 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001441 int codec_width_;
1442 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001443 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001444 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001445 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001446 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001447 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001448 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001449 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001450 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001451};
1452
mflodmancc3d4422017-08-03 08:27:51 -07001453TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001455 DataRate::BitsPerSec(kTargetBitrateBps),
1456 DataRate::BitsPerSec(kTargetBitrateBps),
1457 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001458 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001459 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001460 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001461 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001462 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001463}
1464
mflodmancc3d4422017-08-03 08:27:51 -07001465TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001466 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001467 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001468 // The encoder will cache up to one frame for a short duration. Adding two
1469 // frames means that the first frame will be dropped and the second frame will
1470 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001471 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001472 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001473 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001474 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001475
Henrik Boström381d1092020-05-12 18:49:07 +02001476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001477 DataRate::BitsPerSec(kTargetBitrateBps),
1478 DataRate::BitsPerSec(kTargetBitrateBps),
1479 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001480
Sebastian Janssona3177052018-04-10 13:05:49 +02001481 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001482 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001483 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1484
1485 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001486 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001487}
1488
mflodmancc3d4422017-08-03 08:27:51 -07001489TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001490 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001491 DataRate::BitsPerSec(kTargetBitrateBps),
1492 DataRate::BitsPerSec(kTargetBitrateBps),
1493 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001494 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001495 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001496
Henrik Boström381d1092020-05-12 18:49:07 +02001497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1498 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1499 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001500 // The encoder will cache up to one frame for a short duration. Adding two
1501 // frames means that the first frame will be dropped and the second frame will
1502 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001503 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001504 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001505
Henrik Boström381d1092020-05-12 18:49:07 +02001506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001507 DataRate::BitsPerSec(kTargetBitrateBps),
1508 DataRate::BitsPerSec(kTargetBitrateBps),
1509 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001510 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001511 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1512 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001513 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001514}
1515
mflodmancc3d4422017-08-03 08:27:51 -07001516TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001518 DataRate::BitsPerSec(kTargetBitrateBps),
1519 DataRate::BitsPerSec(kTargetBitrateBps),
1520 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001521 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001522 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001523
1524 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001525 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001526
perkja49cbd32016-09-16 07:53:41 -07001527 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001528 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001529 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001530}
1531
mflodmancc3d4422017-08-03 08:27:51 -07001532TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001534 DataRate::BitsPerSec(kTargetBitrateBps),
1535 DataRate::BitsPerSec(kTargetBitrateBps),
1536 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001537
perkja49cbd32016-09-16 07:53:41 -07001538 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001539 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001540
mflodmancc3d4422017-08-03 08:27:51 -07001541 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001542 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001543 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001544 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1545 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001546}
1547
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001548class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1549 public:
1550 VideoStreamEncoderBlockedTest() {}
1551
1552 TaskQueueFactory* GetTaskQueueFactory() override {
1553 return task_queue_factory_.get();
1554 }
1555
1556 private:
1557 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1558 CreateDefaultTaskQueueFactory();
1559};
1560
1561TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001563 DataRate::BitsPerSec(kTargetBitrateBps),
1564 DataRate::BitsPerSec(kTargetBitrateBps),
1565 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001566
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001567 int dropped_count = 0;
1568 stats_proxy_->SetDroppedFrameCallback(
1569 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1570 ++dropped_count;
1571 });
1572
perkj26091b12016-09-01 01:17:40 -07001573 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001574 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001575 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001576 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1577 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001578 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1579 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001580 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001581 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001582
mflodmancc3d4422017-08-03 08:27:51 -07001583 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001584
1585 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001586}
1587
Henrik Boström56db9ff2021-03-24 09:06:45 +01001588TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001589 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001590 DataRate::BitsPerSec(kTargetBitrateBps),
1591 DataRate::BitsPerSec(kTargetBitrateBps),
1592 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001593
1594 rtc::Event frame_destroyed_event;
1595 video_source_.IncomingCapturedFrame(
1596 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001597 WaitForEncodedFrame(1);
1598 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1599 fake_encoder_.GetLastInputPixelFormat());
1600 EXPECT_EQ(fake_encoder_.codec_config().width,
1601 fake_encoder_.GetLastInputWidth());
1602 EXPECT_EQ(fake_encoder_.codec_config().height,
1603 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001604 video_stream_encoder_->Stop();
1605}
1606
Henrik Boström56db9ff2021-03-24 09:06:45 +01001607TEST_F(VideoStreamEncoderTest,
1608 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001609 // Use the cropping factory.
1610 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001611 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001612 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1613 kMaxPayloadLength);
1614 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1615
1616 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001617 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001618 DataRate::BitsPerSec(kTargetBitrateBps),
1619 DataRate::BitsPerSec(kTargetBitrateBps),
1620 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001621 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1622 WaitForEncodedFrame(1);
1623 // The encoder will have been configured once.
1624 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1625 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1626 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1627
1628 // Now send in a fake frame that needs to be cropped as the width/height
1629 // aren't divisible by 4 (see CreateEncoderStreams above).
1630 rtc::Event frame_destroyed_event;
1631 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1632 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001633 WaitForEncodedFrame(2);
1634 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1635 fake_encoder_.GetLastInputPixelFormat());
1636 EXPECT_EQ(fake_encoder_.codec_config().width,
1637 fake_encoder_.GetLastInputWidth());
1638 EXPECT_EQ(fake_encoder_.codec_config().height,
1639 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001640 video_stream_encoder_->Stop();
1641}
1642
Evan Shrubsole895556e2020-10-05 09:15:13 +02001643TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1645 DataRate::BitsPerSec(kTargetBitrateBps),
1646 DataRate::BitsPerSec(kTargetBitrateBps),
1647 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1648
1649 video_source_.IncomingCapturedFrame(
1650 CreateNV12Frame(1, codec_width_, codec_height_));
1651 WaitForEncodedFrame(1);
1652 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1653 fake_encoder_.GetLastInputPixelFormat());
1654 video_stream_encoder_->Stop();
1655}
1656
Henrik Boström56db9ff2021-03-24 09:06:45 +01001657TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1659 DataRate::BitsPerSec(kTargetBitrateBps),
1660 DataRate::BitsPerSec(kTargetBitrateBps),
1661 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1662
1663 fake_encoder_.SetPreferredPixelFormats({});
1664
1665 rtc::Event frame_destroyed_event;
1666 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1667 1, &frame_destroyed_event, codec_width_, codec_height_));
1668 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001669 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001670 fake_encoder_.GetLastInputPixelFormat());
1671 video_stream_encoder_->Stop();
1672}
1673
Henrik Boström56db9ff2021-03-24 09:06:45 +01001674TEST_F(VideoStreamEncoderTest,
1675 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1677 DataRate::BitsPerSec(kTargetBitrateBps),
1678 DataRate::BitsPerSec(kTargetBitrateBps),
1679 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1680
1681 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1682
1683 rtc::Event frame_destroyed_event;
1684 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1685 1, &frame_destroyed_event, codec_width_, codec_height_));
1686 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001687 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001688 fake_encoder_.GetLastInputPixelFormat());
1689 video_stream_encoder_->Stop();
1690}
1691
Henrik Boström56db9ff2021-03-24 09:06:45 +01001692TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1694 DataRate::BitsPerSec(kTargetBitrateBps),
1695 DataRate::BitsPerSec(kTargetBitrateBps),
1696 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1697
1698 // Fake NV12 native frame does not allow mapping to I444.
1699 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1700
1701 rtc::Event frame_destroyed_event;
1702 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1703 1, &frame_destroyed_event, codec_width_, codec_height_));
1704 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001705 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001706 fake_encoder_.GetLastInputPixelFormat());
1707 video_stream_encoder_->Stop();
1708}
1709
Henrik Boström56db9ff2021-03-24 09:06:45 +01001710TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1712 DataRate::BitsPerSec(kTargetBitrateBps),
1713 DataRate::BitsPerSec(kTargetBitrateBps),
1714 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1715
1716 rtc::Event frame_destroyed_event;
1717 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1718 1, &frame_destroyed_event, codec_width_, codec_height_));
1719 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001720 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001721 fake_encoder_.GetLastInputPixelFormat());
1722 video_stream_encoder_->Stop();
1723}
1724
Ying Wang9b881ab2020-02-07 14:29:32 +01001725TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001727 DataRate::BitsPerSec(kTargetBitrateBps),
1728 DataRate::BitsPerSec(kTargetBitrateBps),
1729 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001730 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1731 WaitForEncodedFrame(1);
1732
Henrik Boström381d1092020-05-12 18:49:07 +02001733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001734 DataRate::BitsPerSec(kTargetBitrateBps),
1735 DataRate::BitsPerSec(kTargetBitrateBps),
1736 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001737 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1738 // frames. Adding two frames means that the first frame will be dropped and
1739 // the second frame will be sent to the encoder.
1740 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1741 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1742 WaitForEncodedFrame(3);
1743 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1744 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1745 WaitForEncodedFrame(5);
1746 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1747 video_stream_encoder_->Stop();
1748}
1749
mflodmancc3d4422017-08-03 08:27:51 -07001750TEST_F(VideoStreamEncoderTest,
1751 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001753 DataRate::BitsPerSec(kTargetBitrateBps),
1754 DataRate::BitsPerSec(kTargetBitrateBps),
1755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001756 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001757
1758 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001759 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001760 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001761 // The encoder will have been configured once when the first frame is
1762 // received.
1763 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001764
1765 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001766 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001767 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001768 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001769 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001770
1771 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001772 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001773 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001774 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001775 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001776
mflodmancc3d4422017-08-03 08:27:51 -07001777 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001778}
1779
mflodmancc3d4422017-08-03 08:27:51 -07001780TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001782 DataRate::BitsPerSec(kTargetBitrateBps),
1783 DataRate::BitsPerSec(kTargetBitrateBps),
1784 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001785
1786 // Capture a frame and wait for it to synchronize with the encoder thread.
1787 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001788 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001789 // The encoder will have been configured once.
1790 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001791 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1792 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1793
1794 codec_width_ *= 2;
1795 codec_height_ *= 2;
1796 // Capture a frame with a higher resolution and wait for it to synchronize
1797 // with the encoder thread.
1798 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001799 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001800 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1801 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001802 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001803
mflodmancc3d4422017-08-03 08:27:51 -07001804 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001805}
1806
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001807TEST_F(VideoStreamEncoderTest,
1808 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001810 DataRate::BitsPerSec(kTargetBitrateBps),
1811 DataRate::BitsPerSec(kTargetBitrateBps),
1812 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001813
1814 // Capture a frame and wait for it to synchronize with the encoder thread.
1815 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1816 WaitForEncodedFrame(1);
1817
1818 VideoEncoderConfig video_encoder_config;
1819 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1820 // Changing the max payload data length recreates encoder.
1821 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1822 kMaxPayloadLength / 2);
1823
1824 // Capture a frame and wait for it to synchronize with the encoder thread.
1825 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1826 WaitForEncodedFrame(2);
1827 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1828
1829 video_stream_encoder_->Stop();
1830}
1831
Sergey Silkin5ee69672019-07-02 14:18:34 +02001832TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001834 DataRate::BitsPerSec(kTargetBitrateBps),
1835 DataRate::BitsPerSec(kTargetBitrateBps),
1836 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001837
1838 VideoEncoderConfig video_encoder_config;
1839 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1840 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1841 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1842 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1843 kMaxPayloadLength);
1844
1845 // Capture a frame and wait for it to synchronize with the encoder thread.
1846 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1847 WaitForEncodedFrame(1);
1848 // The encoder will have been configured once when the first frame is
1849 // received.
1850 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1851 EXPECT_EQ(kTargetBitrateBps,
1852 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1853 EXPECT_EQ(kStartBitrateBps,
1854 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1855
Sergey Silkin6456e352019-07-08 17:56:40 +02001856 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1857 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001858 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1859 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1860 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1861 kMaxPayloadLength);
1862
1863 // Capture a frame and wait for it to synchronize with the encoder thread.
1864 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1865 WaitForEncodedFrame(2);
1866 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1867 // Bitrate limits have changed - rate allocator should be reconfigured,
1868 // encoder should not be reconfigured.
1869 EXPECT_EQ(kTargetBitrateBps * 2,
1870 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1871 EXPECT_EQ(kStartBitrateBps * 2,
1872 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1873 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1874
1875 video_stream_encoder_->Stop();
1876}
1877
Sergey Silkin6456e352019-07-08 17:56:40 +02001878TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001879 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001881 DataRate::BitsPerSec(kTargetBitrateBps),
1882 DataRate::BitsPerSec(kTargetBitrateBps),
1883 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001884
Sergey Silkincd02eba2020-01-20 14:48:40 +01001885 const uint32_t kMinEncBitrateKbps = 100;
1886 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001887 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001888 /*frame_size_pixels=*/codec_width_ * codec_height_,
1889 /*min_start_bitrate_bps=*/0,
1890 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1891 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001892 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1893
Sergey Silkincd02eba2020-01-20 14:48:40 +01001894 VideoEncoderConfig video_encoder_config;
1895 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1896 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1897 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1898 (kMinEncBitrateKbps + 1) * 1000;
1899 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1900 kMaxPayloadLength);
1901
1902 // When both encoder and app provide bitrate limits, the intersection of
1903 // provided sets should be used.
1904 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1905 WaitForEncodedFrame(1);
1906 EXPECT_EQ(kMaxEncBitrateKbps,
1907 bitrate_allocator_factory_.codec_config().maxBitrate);
1908 EXPECT_EQ(kMinEncBitrateKbps + 1,
1909 bitrate_allocator_factory_.codec_config().minBitrate);
1910
1911 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1912 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1913 (kMinEncBitrateKbps - 1) * 1000;
1914 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1915 kMaxPayloadLength);
1916 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001917 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001918 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001919 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001920 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001921 bitrate_allocator_factory_.codec_config().minBitrate);
1922
Sergey Silkincd02eba2020-01-20 14:48:40 +01001923 video_stream_encoder_->Stop();
1924}
1925
1926TEST_F(VideoStreamEncoderTest,
1927 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001928 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001929 DataRate::BitsPerSec(kTargetBitrateBps),
1930 DataRate::BitsPerSec(kTargetBitrateBps),
1931 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001932
1933 const uint32_t kMinAppBitrateKbps = 100;
1934 const uint32_t kMaxAppBitrateKbps = 200;
1935 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1936 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1937 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1938 /*frame_size_pixels=*/codec_width_ * codec_height_,
1939 /*min_start_bitrate_bps=*/0,
1940 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1941 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1942 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1943
1944 VideoEncoderConfig video_encoder_config;
1945 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1946 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1947 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1948 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1950 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001951
Sergey Silkincd02eba2020-01-20 14:48:40 +01001952 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1953 WaitForEncodedFrame(1);
1954 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001955 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001956 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001957 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001958
1959 video_stream_encoder_->Stop();
1960}
1961
1962TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001963 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001965 DataRate::BitsPerSec(kTargetBitrateBps),
1966 DataRate::BitsPerSec(kTargetBitrateBps),
1967 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001968
1969 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001970 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001971 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001973 fake_encoder_.SetResolutionBitrateLimits(
1974 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1975
1976 VideoEncoderConfig video_encoder_config;
1977 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1978 video_encoder_config.max_bitrate_bps = 0;
1979 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1980 kMaxPayloadLength);
1981
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001982 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001983 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1984 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001985 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1986 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001987 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1988 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1989
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001991 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1992 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001993 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1994 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001995 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1997
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001998 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001999 // encoder for 360p should be used.
2000 video_source_.IncomingCapturedFrame(
2001 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2002 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002003 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2004 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002005 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2006 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2007
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002008 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002009 // ignored.
2010 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2011 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002012 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2013 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002014 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2015 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002016 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2017 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002018 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2019 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2020
2021 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2022 // for 270p should be used.
2023 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2024 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2026 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002027 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2028 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2029
2030 video_stream_encoder_->Stop();
2031}
2032
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002033TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002035 DataRate::BitsPerSec(kTargetBitrateBps),
2036 DataRate::BitsPerSec(kTargetBitrateBps),
2037 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002038
2039 VideoEncoderConfig video_encoder_config;
2040 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2041 video_encoder_config.max_bitrate_bps = 0;
2042 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2043 kMaxPayloadLength);
2044
2045 // Encode 720p frame to get the default encoder target bitrate.
2046 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2047 WaitForEncodedFrame(1);
2048 const uint32_t kDefaultTargetBitrateFor720pKbps =
2049 bitrate_allocator_factory_.codec_config()
2050 .simulcastStream[0]
2051 .targetBitrate;
2052
2053 // Set the max recommended encoder bitrate to something lower than the default
2054 // target bitrate.
2055 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2056 1280 * 720, 10 * 1000, 10 * 1000,
2057 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2058 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2059
2060 // Change resolution to trigger encoder reinitialization.
2061 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2062 WaitForEncodedFrame(2);
2063 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2064 WaitForEncodedFrame(3);
2065
2066 // Ensure the target bitrate is capped by the max bitrate.
2067 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2068 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2069 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2070 .simulcastStream[0]
2071 .targetBitrate *
2072 1000,
2073 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2074
2075 video_stream_encoder_->Stop();
2076}
2077
Åsa Perssona7e34d32021-01-20 15:36:13 +01002078TEST_F(VideoStreamEncoderTest,
2079 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2080 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2081 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2082 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2083 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2084 fake_encoder_.SetResolutionBitrateLimits(
2085 {kEncoderLimits270p, kEncoderLimits360p});
2086
2087 // Two streams, highest stream active.
2088 VideoEncoderConfig config;
2089 const int kNumStreams = 2;
2090 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2091 config.max_bitrate_bps = 0;
2092 config.simulcast_layers[0].active = false;
2093 config.simulcast_layers[1].active = true;
2094 config.video_stream_factory =
2095 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2096 "VP8", /*max qp*/ 56, /*screencast*/ false,
2097 /*screenshare enabled*/ false);
2098 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2099
2100 // The encoder bitrate limits for 270p should be used.
2101 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2102 EXPECT_FALSE(WaitForFrame(1000));
2103 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2104 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2105 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2106 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2107 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2108
2109 // The encoder bitrate limits for 360p should be used.
2110 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2111 EXPECT_FALSE(WaitForFrame(1000));
2112 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2113 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2114 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2115 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2116
2117 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2118 video_source_.IncomingCapturedFrame(
2119 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2120 EXPECT_FALSE(WaitForFrame(1000));
2121 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2122 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2123 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2124 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2125
2126 // Resolution higher than 360p. Encoder limits should be ignored.
2127 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2128 EXPECT_FALSE(WaitForFrame(1000));
2129 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2130 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2131 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2132 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2133 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2134 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2135 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2136 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2137
2138 // Resolution lower than 270p. The encoder limits for 270p should be used.
2139 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2140 EXPECT_FALSE(WaitForFrame(1000));
2141 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2142 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2143 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2144 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2145
2146 video_stream_encoder_->Stop();
2147}
2148
2149TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002150 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2151 // Two streams, highest stream active.
2152 VideoEncoderConfig config;
2153 const int kNumStreams = 2;
2154 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2155 config.max_bitrate_bps = 0;
2156 config.simulcast_layers[0].active = false;
2157 config.simulcast_layers[1].active = true;
2158 config.video_stream_factory =
2159 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2160 "VP8", /*max qp*/ 56, /*screencast*/ false,
2161 /*screenshare enabled*/ false);
2162 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2163
2164 // Default bitrate limits for 270p should be used.
2165 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2166 kDefaultLimits270p =
2167 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002168 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002169 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2170 EXPECT_FALSE(WaitForFrame(1000));
2171 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2172 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2173 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2174 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2175 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2176
2177 // Default bitrate limits for 360p should be used.
2178 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2179 kDefaultLimits360p =
2180 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002181 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002182 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2183 EXPECT_FALSE(WaitForFrame(1000));
2184 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2185 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2186 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2187 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2188
2189 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2190 video_source_.IncomingCapturedFrame(
2191 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2192 EXPECT_FALSE(WaitForFrame(1000));
2193 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2194 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2195 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2196 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2197
2198 // Default bitrate limits for 540p should be used.
2199 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2200 kDefaultLimits540p =
2201 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002202 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002203 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2204 EXPECT_FALSE(WaitForFrame(1000));
2205 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2206 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2207 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2208 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2209
2210 video_stream_encoder_->Stop();
2211}
2212
2213TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2215 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2216 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2217 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2218 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2219 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2220 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2221 fake_encoder_.SetResolutionBitrateLimits(
2222 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2223
2224 // Three streams, middle stream active.
2225 VideoEncoderConfig config;
2226 const int kNumStreams = 3;
2227 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2228 config.simulcast_layers[0].active = false;
2229 config.simulcast_layers[1].active = true;
2230 config.simulcast_layers[2].active = false;
2231 config.video_stream_factory =
2232 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2233 "VP8", /*max qp*/ 56, /*screencast*/ false,
2234 /*screenshare enabled*/ false);
2235 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2236
2237 // The encoder bitrate limits for 360p should be used.
2238 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2239 EXPECT_FALSE(WaitForFrame(1000));
2240 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2242 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2244 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2245
2246 // The encoder bitrate limits for 270p should be used.
2247 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2248 EXPECT_FALSE(WaitForFrame(1000));
2249 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2250 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2251 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2252 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2253
2254 video_stream_encoder_->Stop();
2255}
2256
2257TEST_F(VideoStreamEncoderTest,
2258 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2259 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2260 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2261 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2262 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2263 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2264 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2265 fake_encoder_.SetResolutionBitrateLimits(
2266 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2267
2268 // Three streams, lowest stream active.
2269 VideoEncoderConfig config;
2270 const int kNumStreams = 3;
2271 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2272 config.simulcast_layers[0].active = true;
2273 config.simulcast_layers[1].active = false;
2274 config.simulcast_layers[2].active = false;
2275 config.video_stream_factory =
2276 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2277 "VP8", /*max qp*/ 56, /*screencast*/ false,
2278 /*screenshare enabled*/ false);
2279 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2280
2281 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2282 // on lowest stream, limits for 270p should not be used
2283 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2284 EXPECT_FALSE(WaitForFrame(1000));
2285 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2286 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2287 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2288 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2289 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2290
2291 video_stream_encoder_->Stop();
2292}
2293
2294TEST_F(VideoStreamEncoderTest,
2295 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2296 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2297 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2298 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2299 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2300 fake_encoder_.SetResolutionBitrateLimits(
2301 {kEncoderLimits270p, kEncoderLimits360p});
2302 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2303
2304 // Two streams, highest stream active.
2305 VideoEncoderConfig config;
2306 const int kNumStreams = 2;
2307 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2308 config.simulcast_layers[0].active = false;
2309 config.simulcast_layers[1].active = true;
2310 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2311 config.video_stream_factory =
2312 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2313 "VP8", /*max qp*/ 56, /*screencast*/ false,
2314 /*screenshare enabled*/ false);
2315 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2316
2317 // The encoder bitrate limits for 270p should be used.
2318 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2319 EXPECT_FALSE(WaitForFrame(1000));
2320 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2321 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2322 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2323 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2324 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2325
2326 // The max configured bitrate is less than the encoder limit for 360p.
2327 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2328 EXPECT_FALSE(WaitForFrame(1000));
2329 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2330 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2331 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2332 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2333
2334 video_stream_encoder_->Stop();
2335}
2336
mflodmancc3d4422017-08-03 08:27:51 -07002337TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002338 EXPECT_TRUE(video_source_.has_sinks());
2339 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002340 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002341 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002342 EXPECT_FALSE(video_source_.has_sinks());
2343 EXPECT_TRUE(new_video_source.has_sinks());
2344
mflodmancc3d4422017-08-03 08:27:51 -07002345 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002346}
2347
mflodmancc3d4422017-08-03 08:27:51 -07002348TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002349 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002350 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002351 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002352 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002353}
2354
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002355class ResolutionAlignmentTest
2356 : public VideoStreamEncoderTest,
2357 public ::testing::WithParamInterface<
2358 ::testing::tuple<int, std::vector<double>>> {
2359 public:
2360 ResolutionAlignmentTest()
2361 : requested_alignment_(::testing::get<0>(GetParam())),
2362 scale_factors_(::testing::get<1>(GetParam())) {}
2363
2364 protected:
2365 const int requested_alignment_;
2366 const std::vector<double> scale_factors_;
2367};
2368
2369INSTANTIATE_TEST_SUITE_P(
2370 AlignmentAndScaleFactors,
2371 ResolutionAlignmentTest,
2372 ::testing::Combine(
2373 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2374 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2375 std::vector<double>{-1.0, -1.0},
2376 std::vector<double>{-1.0, -1.0, -1.0},
2377 std::vector<double>{4.0, 2.0, 1.0},
2378 std::vector<double>{9999.0, -1.0, 1.0},
2379 std::vector<double>{3.99, 2.01, 1.0},
2380 std::vector<double>{4.9, 1.7, 1.25},
2381 std::vector<double>{10.0, 4.0, 3.0},
2382 std::vector<double>{1.75, 3.5},
2383 std::vector<double>{1.5, 2.5},
2384 std::vector<double>{1.3, 1.0})));
2385
2386TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2387 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002388 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002389 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2390 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2391
2392 // Fill config with the scaling factor by which to reduce encoding size.
2393 const int num_streams = scale_factors_.size();
2394 VideoEncoderConfig config;
2395 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2396 for (int i = 0; i < num_streams; ++i) {
2397 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2398 }
2399 config.video_stream_factory =
2400 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2401 "VP8", /*max qp*/ 56, /*screencast*/ false,
2402 /*screenshare enabled*/ false);
2403 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2404
Henrik Boström381d1092020-05-12 18:49:07 +02002405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002406 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2407 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2408 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2409 // Wait for all layers before triggering event.
2410 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002411
2412 // On the 1st frame, we should have initialized the encoder and
2413 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002414 int64_t timestamp_ms = kFrameIntervalMs;
2415 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2416 WaitForEncodedFrame(timestamp_ms);
2417 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002418
2419 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2420 // (It's up the to the encoder to potentially drop the previous frame,
2421 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002422 timestamp_ms += kFrameIntervalMs;
2423 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2424 WaitForEncodedFrame(timestamp_ms);
2425 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2426
2427 VideoCodec codec = fake_encoder_.video_codec();
2428 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2429 // Frame size should be a multiple of the requested alignment.
2430 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2431 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2432 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2433 // Aspect ratio should match.
2434 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2435 codec.height * codec.simulcastStream[i].width);
2436 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002437
2438 video_stream_encoder_->Stop();
2439}
2440
Jonathan Yubc771b72017-12-08 17:04:29 -08002441TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2442 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002443 const int kWidth = 1280;
2444 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002445
2446 // We rely on the automatic resolution adaptation, but we handle framerate
2447 // adaptation manually by mocking the stats proxy.
2448 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002449
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002450 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002452 DataRate::BitsPerSec(kTargetBitrateBps),
2453 DataRate::BitsPerSec(kTargetBitrateBps),
2454 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002455 video_stream_encoder_->SetSource(&video_source_,
2456 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002457 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002458 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002459 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2461
Jonathan Yubc771b72017-12-08 17:04:29 -08002462 // Adapt down as far as possible.
2463 rtc::VideoSinkWants last_wants;
2464 int64_t t = 1;
2465 int loop_count = 0;
2466 do {
2467 ++loop_count;
2468 last_wants = video_source_.sink_wants();
2469
2470 // Simulate the framerate we've been asked to adapt to.
2471 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2472 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2473 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2474 mock_stats.input_frame_rate = fps;
2475 stats_proxy_->SetMockStats(mock_stats);
2476
2477 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2478 sink_.WaitForEncodedFrame(t);
2479 t += frame_interval_ms;
2480
mflodmancc3d4422017-08-03 08:27:51 -07002481 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002482 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002483 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002484 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2485 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002486 } while (video_source_.sink_wants().max_pixel_count <
2487 last_wants.max_pixel_count ||
2488 video_source_.sink_wants().max_framerate_fps <
2489 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002490
Jonathan Yubc771b72017-12-08 17:04:29 -08002491 // Verify that we've adapted all the way down.
2492 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002493 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002494 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2495 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002496 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002497 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2498 *video_source_.last_sent_height());
2499 EXPECT_EQ(kMinBalancedFramerateFps,
2500 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002501
Jonathan Yubc771b72017-12-08 17:04:29 -08002502 // Adapt back up the same number of times we adapted down.
2503 for (int i = 0; i < loop_count - 1; ++i) {
2504 last_wants = video_source_.sink_wants();
2505
2506 // Simulate the framerate we've been asked to adapt to.
2507 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2508 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2509 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2510 mock_stats.input_frame_rate = fps;
2511 stats_proxy_->SetMockStats(mock_stats);
2512
2513 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2514 sink_.WaitForEncodedFrame(t);
2515 t += frame_interval_ms;
2516
Henrik Boström91aa7322020-04-28 12:24:33 +02002517 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002518 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002519 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002520 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2521 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002522 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2523 last_wants.max_pixel_count ||
2524 video_source_.sink_wants().max_framerate_fps >
2525 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002526 }
2527
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002528 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002529 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002530 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002531 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2532 EXPECT_EQ((loop_count - 1) * 2,
2533 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002534
mflodmancc3d4422017-08-03 08:27:51 -07002535 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002536}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002537
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002538TEST_F(VideoStreamEncoderTest,
2539 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2540 video_stream_encoder_->OnBitrateUpdated(
2541 DataRate::BitsPerSec(kTargetBitrateBps),
2542 DataRate::BitsPerSec(kTargetBitrateBps),
2543 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002544 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002545
2546 const int kFrameWidth = 1280;
2547 const int kFrameHeight = 720;
2548
2549 int64_t ntp_time = kFrameIntervalMs;
2550
2551 // Force an input frame rate to be available, or the adaptation call won't
2552 // know what framerate to adapt form.
2553 const int kInputFps = 30;
2554 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2555 stats.input_frame_rate = kInputFps;
2556 stats_proxy_->SetMockStats(stats);
2557
2558 video_source_.set_adaptation_enabled(true);
2559 video_stream_encoder_->SetSource(
2560 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002561 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002562 video_source_.IncomingCapturedFrame(
2563 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2564 sink_.WaitForEncodedFrame(ntp_time);
2565 ntp_time += kFrameIntervalMs;
2566
2567 // Trigger CPU overuse.
2568 video_stream_encoder_->TriggerCpuOveruse();
2569 video_source_.IncomingCapturedFrame(
2570 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2571 sink_.WaitForEncodedFrame(ntp_time);
2572 ntp_time += kFrameIntervalMs;
2573
2574 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2575 EXPECT_EQ(std::numeric_limits<int>::max(),
2576 video_source_.sink_wants().max_pixel_count);
2577 // Some framerate constraint should be set.
2578 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2579 EXPECT_LT(restricted_fps, kInputFps);
2580 video_source_.IncomingCapturedFrame(
2581 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2582 sink_.WaitForEncodedFrame(ntp_time);
2583 ntp_time += 100;
2584
Henrik Boström2671dac2020-05-19 16:29:09 +02002585 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002586 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2587 // Give the encoder queue time to process the change in degradation preference
2588 // by waiting for an encoded frame.
2589 video_source_.IncomingCapturedFrame(
2590 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2591 sink_.WaitForEncodedFrame(ntp_time);
2592 ntp_time += kFrameIntervalMs;
2593
2594 video_stream_encoder_->TriggerQualityLow();
2595 video_source_.IncomingCapturedFrame(
2596 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2597 sink_.WaitForEncodedFrame(ntp_time);
2598 ntp_time += kFrameIntervalMs;
2599
2600 // Some resolution constraint should be set.
2601 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2602 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2603 kFrameWidth * kFrameHeight);
2604 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2605
2606 int pixel_count = video_source_.sink_wants().max_pixel_count;
2607 // Triggering a CPU underuse should not change the sink wants since it has
2608 // not been overused for resolution since we changed degradation preference.
2609 video_stream_encoder_->TriggerCpuUnderuse();
2610 video_source_.IncomingCapturedFrame(
2611 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2612 sink_.WaitForEncodedFrame(ntp_time);
2613 ntp_time += kFrameIntervalMs;
2614 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2615 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2616
Evan Shrubsole64469032020-06-11 10:45:29 +02002617 // Change the degradation preference back. CPU underuse should not adapt since
2618 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002619 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002620 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2621 video_source_.IncomingCapturedFrame(
2622 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2623 sink_.WaitForEncodedFrame(ntp_time);
2624 ntp_time += 100;
2625 // Resolution adaptations is gone after changing degradation preference.
2626 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2627 EXPECT_EQ(std::numeric_limits<int>::max(),
2628 video_source_.sink_wants().max_pixel_count);
2629 // The fps adaptation from above is now back.
2630 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2631
2632 // Trigger CPU underuse.
2633 video_stream_encoder_->TriggerCpuUnderuse();
2634 video_source_.IncomingCapturedFrame(
2635 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2636 sink_.WaitForEncodedFrame(ntp_time);
2637 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002638 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2639
2640 // Trigger QP underuse, fps should return to normal.
2641 video_stream_encoder_->TriggerQualityHigh();
2642 video_source_.IncomingCapturedFrame(
2643 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2644 sink_.WaitForEncodedFrame(ntp_time);
2645 ntp_time += kFrameIntervalMs;
2646 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002647
2648 video_stream_encoder_->Stop();
2649}
2650
mflodmancc3d4422017-08-03 08:27:51 -07002651TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002653 DataRate::BitsPerSec(kTargetBitrateBps),
2654 DataRate::BitsPerSec(kTargetBitrateBps),
2655 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002656 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002657
sprangc5d62e22017-04-02 23:53:04 -07002658 const int kFrameWidth = 1280;
2659 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002660
Åsa Persson8c1bf952018-09-13 10:42:19 +02002661 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002662
kthelgason5e13d412016-12-01 03:59:51 -08002663 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002664 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002665 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002666 frame_timestamp += kFrameIntervalMs;
2667
perkj803d97f2016-11-01 11:45:46 -07002668 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002669 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002670 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002671 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002673 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002674
asapersson0944a802017-04-07 00:57:58 -07002675 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002676 // wanted resolution.
2677 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2678 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2679 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002680 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002681
2682 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002683 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002684 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002685 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002686 // Give the encoder queue time to process the change in degradation preference
2687 // by waiting for an encoded frame.
2688 new_video_source.IncomingCapturedFrame(
2689 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2690 sink_.WaitForEncodedFrame(frame_timestamp);
2691 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002692 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002693 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002694
sprangc5d62e22017-04-02 23:53:04 -07002695 // Force an input frame rate to be available, or the adaptation call won't
2696 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002697 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002698 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002699 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002700 stats_proxy_->SetMockStats(stats);
2701
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002703 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002704 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002705 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002706 frame_timestamp += kFrameIntervalMs;
2707
2708 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002709 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002710 EXPECT_EQ(std::numeric_limits<int>::max(),
2711 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002712 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002713
asapersson02465b82017-04-10 01:12:52 -07002714 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002715 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2716 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002717 // Give the encoder queue time to process the change in degradation preference
2718 // by waiting for an encoded frame.
2719 new_video_source.IncomingCapturedFrame(
2720 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2721 sink_.WaitForEncodedFrame(frame_timestamp);
2722 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002723 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002724
mflodmancc3d4422017-08-03 08:27:51 -07002725 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002726 new_video_source.IncomingCapturedFrame(
2727 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002728 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002729 frame_timestamp += kFrameIntervalMs;
2730
2731 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002732 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002733
2734 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002735 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002736 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002737 // Give the encoder queue time to process the change in degradation preference
2738 // by waiting for an encoded frame.
2739 new_video_source.IncomingCapturedFrame(
2740 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2741 sink_.WaitForEncodedFrame(frame_timestamp);
2742 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002743 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2744 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002745 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002746 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002747
2748 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002750 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002751 // Give the encoder queue time to process the change in degradation preference
2752 // by waiting for an encoded frame.
2753 new_video_source.IncomingCapturedFrame(
2754 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2755 sink_.WaitForEncodedFrame(frame_timestamp);
2756 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002757 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2758 EXPECT_EQ(std::numeric_limits<int>::max(),
2759 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002760 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002761
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002763}
2764
mflodmancc3d4422017-08-03 08:27:51 -07002765TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002767 DataRate::BitsPerSec(kTargetBitrateBps),
2768 DataRate::BitsPerSec(kTargetBitrateBps),
2769 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002770
asaperssonfab67072017-04-04 05:51:49 -07002771 const int kWidth = 1280;
2772 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002773 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002774 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002775 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2776 EXPECT_FALSE(stats.bw_limited_resolution);
2777 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2778
2779 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002780 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002781 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002782 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002783
2784 stats = stats_proxy_->GetStats();
2785 EXPECT_TRUE(stats.bw_limited_resolution);
2786 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2787
2788 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002789 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002790 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002791 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002792
2793 stats = stats_proxy_->GetStats();
2794 EXPECT_FALSE(stats.bw_limited_resolution);
2795 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2796 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2797
mflodmancc3d4422017-08-03 08:27:51 -07002798 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002799}
2800
mflodmancc3d4422017-08-03 08:27:51 -07002801TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002803 DataRate::BitsPerSec(kTargetBitrateBps),
2804 DataRate::BitsPerSec(kTargetBitrateBps),
2805 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002806
2807 const int kWidth = 1280;
2808 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002809 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002810 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002811 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2812 EXPECT_FALSE(stats.cpu_limited_resolution);
2813 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2814
2815 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002816 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002817 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002818 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002819
2820 stats = stats_proxy_->GetStats();
2821 EXPECT_TRUE(stats.cpu_limited_resolution);
2822 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2823
2824 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002825 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002826 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002827 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002828
2829 stats = stats_proxy_->GetStats();
2830 EXPECT_FALSE(stats.cpu_limited_resolution);
2831 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002832 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002833
mflodmancc3d4422017-08-03 08:27:51 -07002834 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002835}
2836
mflodmancc3d4422017-08-03 08:27:51 -07002837TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002839 DataRate::BitsPerSec(kTargetBitrateBps),
2840 DataRate::BitsPerSec(kTargetBitrateBps),
2841 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002842
asaperssonfab67072017-04-04 05:51:49 -07002843 const int kWidth = 1280;
2844 const int kHeight = 720;
2845 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002846 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002847 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002848 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002849 EXPECT_FALSE(stats.cpu_limited_resolution);
2850 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2851
asaperssonfab67072017-04-04 05:51:49 -07002852 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002854 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002855 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002856 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002857 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002858 EXPECT_TRUE(stats.cpu_limited_resolution);
2859 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2860
2861 // Set new source with adaptation still enabled.
2862 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002863 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002864 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002865
asaperssonfab67072017-04-04 05:51:49 -07002866 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002867 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002868 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002869 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002870 EXPECT_TRUE(stats.cpu_limited_resolution);
2871 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2872
2873 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002874 video_stream_encoder_->SetSource(&new_video_source,
2875 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002876
asaperssonfab67072017-04-04 05:51:49 -07002877 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002878 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002879 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002880 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002881 EXPECT_FALSE(stats.cpu_limited_resolution);
2882 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2883
2884 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002885 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002886 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002887
asaperssonfab67072017-04-04 05:51:49 -07002888 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002889 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002890 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002891 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002892 EXPECT_TRUE(stats.cpu_limited_resolution);
2893 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2894
asaperssonfab67072017-04-04 05:51:49 -07002895 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002896 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002897 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002898 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002899 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002900 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002901 EXPECT_FALSE(stats.cpu_limited_resolution);
2902 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002903 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002904
mflodmancc3d4422017-08-03 08:27:51 -07002905 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002906}
2907
mflodmancc3d4422017-08-03 08:27:51 -07002908TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002910 DataRate::BitsPerSec(kTargetBitrateBps),
2911 DataRate::BitsPerSec(kTargetBitrateBps),
2912 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002913
asaperssonfab67072017-04-04 05:51:49 -07002914 const int kWidth = 1280;
2915 const int kHeight = 720;
2916 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002917 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002918 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002919 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002920 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002921 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002922
2923 // Set new source with adaptation still enabled.
2924 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002925 video_stream_encoder_->SetSource(&new_video_source,
2926 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002927
asaperssonfab67072017-04-04 05:51:49 -07002928 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002929 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002930 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002931 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002932 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002933 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002934
asaperssonfab67072017-04-04 05:51:49 -07002935 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002936 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002937 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002938 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002939 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002940 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002941 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002942 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002943
asaperssonfab67072017-04-04 05:51:49 -07002944 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002945 video_stream_encoder_->SetSource(&new_video_source,
2946 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002947
asaperssonfab67072017-04-04 05:51:49 -07002948 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002949 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002950 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002951 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002952 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002953 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002954
asapersson02465b82017-04-10 01:12:52 -07002955 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002956 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002957 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002958
asaperssonfab67072017-04-04 05:51:49 -07002959 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002960 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002961 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002962 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002963 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002964 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2965 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002966
mflodmancc3d4422017-08-03 08:27:51 -07002967 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002968}
2969
mflodmancc3d4422017-08-03 08:27:51 -07002970TEST_F(VideoStreamEncoderTest,
2971 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002973 DataRate::BitsPerSec(kTargetBitrateBps),
2974 DataRate::BitsPerSec(kTargetBitrateBps),
2975 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002976
2977 const int kWidth = 1280;
2978 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002979 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002980 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002981 video_source_.IncomingCapturedFrame(
2982 CreateFrame(timestamp_ms, kWidth, kHeight));
2983 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002984 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2985 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2986 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2987
2988 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002989 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002990 timestamp_ms += kFrameIntervalMs;
2991 video_source_.IncomingCapturedFrame(
2992 CreateFrame(timestamp_ms, kWidth, kHeight));
2993 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002994 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2995 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2996 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2997
2998 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002999 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003000 timestamp_ms += kFrameIntervalMs;
3001 video_source_.IncomingCapturedFrame(
3002 CreateFrame(timestamp_ms, kWidth, kHeight));
3003 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003004 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3005 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3006 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3007
Niels Möller4db138e2018-04-19 09:04:13 +02003008 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003009 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003010
3011 VideoEncoderConfig video_encoder_config;
3012 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3013 // Make format different, to force recreation of encoder.
3014 video_encoder_config.video_format.parameters["foo"] = "foo";
3015 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003016 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003017 timestamp_ms += kFrameIntervalMs;
3018 video_source_.IncomingCapturedFrame(
3019 CreateFrame(timestamp_ms, kWidth, kHeight));
3020 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003021 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3022 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3023 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3024
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003026}
3027
mflodmancc3d4422017-08-03 08:27:51 -07003028TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003029 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003031 DataRate::BitsPerSec(kTargetBitrateBps),
3032 DataRate::BitsPerSec(kTargetBitrateBps),
3033 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3034
3035 const int kWidth = 1280;
3036 const int kHeight = 720;
3037 int sequence = 1;
3038
3039 // Enable BALANCED preference, no initial limitation.
3040 test::FrameForwarder source;
3041 video_stream_encoder_->SetSource(&source,
3042 webrtc::DegradationPreference::BALANCED);
3043 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3044 WaitForEncodedFrame(sequence++);
3045 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3046 EXPECT_FALSE(stats.cpu_limited_resolution);
3047 EXPECT_FALSE(stats.cpu_limited_framerate);
3048 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3049
3050 // Trigger CPU overuse, should now adapt down.
3051 video_stream_encoder_->TriggerCpuOveruse();
3052 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3053 WaitForEncodedFrame(sequence++);
3054 stats = stats_proxy_->GetStats();
3055 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3056
3057 // Set new degradation preference should clear restrictions since we changed
3058 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003059 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003060 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3061 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3062 WaitForEncodedFrame(sequence++);
3063 stats = stats_proxy_->GetStats();
3064 EXPECT_FALSE(stats.cpu_limited_resolution);
3065 EXPECT_FALSE(stats.cpu_limited_framerate);
3066 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3067
3068 // Force an input frame rate to be available, or the adaptation call won't
3069 // know what framerate to adapt from.
3070 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3071 mock_stats.input_frame_rate = 30;
3072 stats_proxy_->SetMockStats(mock_stats);
3073 video_stream_encoder_->TriggerCpuOveruse();
3074 stats_proxy_->ResetMockStats();
3075 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3076 WaitForEncodedFrame(sequence++);
3077
3078 // We have now adapted once.
3079 stats = stats_proxy_->GetStats();
3080 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3081
3082 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003083 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3084 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003085 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3086 WaitForEncodedFrame(sequence++);
3087 stats = stats_proxy_->GetStats();
3088 EXPECT_FALSE(stats.cpu_limited_resolution);
3089 EXPECT_FALSE(stats.cpu_limited_framerate);
3090 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3091
3092 video_stream_encoder_->Stop();
3093}
3094
3095TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003096 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003098 DataRate::BitsPerSec(kTargetBitrateBps),
3099 DataRate::BitsPerSec(kTargetBitrateBps),
3100 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003101
asapersson0944a802017-04-07 00:57:58 -07003102 const int kWidth = 1280;
3103 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003104 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003105
asaperssonfab67072017-04-04 05:51:49 -07003106 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003107 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003108 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003109 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003110 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003111 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3112
asapersson02465b82017-04-10 01:12:52 -07003113 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003114 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003115 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003116 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003117 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003118 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003119 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003120 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3121
3122 // Set new source with adaptation still enabled.
3123 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003124 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003125 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003126
3127 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003128 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003129 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003130 stats = stats_proxy_->GetStats();
3131 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003132 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003133 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3134
sprangc5d62e22017-04-02 23:53:04 -07003135 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003136 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003137 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003138 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003139 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003140 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003141 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003142 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003143 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003144 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003145 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3146
sprangc5d62e22017-04-02 23:53:04 -07003147 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003148 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003149 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3150 mock_stats.input_frame_rate = 30;
3151 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003152 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003153 stats_proxy_->ResetMockStats();
3154
3155 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003156 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003157 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003158
3159 // Framerate now adapted.
3160 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003161 EXPECT_FALSE(stats.cpu_limited_resolution);
3162 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003163 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3164
3165 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003166 video_stream_encoder_->SetSource(&new_video_source,
3167 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003168 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003169 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003170 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003171
3172 stats = stats_proxy_->GetStats();
3173 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003174 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003175 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3176
3177 // Try to trigger overuse. Should not succeed.
3178 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003179 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003180 stats_proxy_->ResetMockStats();
3181
3182 stats = stats_proxy_->GetStats();
3183 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003184 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003185 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3186
3187 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003188 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003189 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003190 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003191 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003192 stats = stats_proxy_->GetStats();
3193 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003194 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003195 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003196
3197 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003198 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003199 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003200 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003201 stats = stats_proxy_->GetStats();
3202 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003203 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003204 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3205
3206 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003208 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003209 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003210 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003211 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003212 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003213 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003214 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003215 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003216 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3217
3218 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003219 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003220 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003221 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003222 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003223 stats = stats_proxy_->GetStats();
3224 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003225 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003226 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003227 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003228
mflodmancc3d4422017-08-03 08:27:51 -07003229 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003230}
3231
mflodmancc3d4422017-08-03 08:27:51 -07003232TEST_F(VideoStreamEncoderTest,
3233 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003234 const int kWidth = 1280;
3235 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003236 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003237 DataRate::BitsPerSec(kTargetBitrateBps),
3238 DataRate::BitsPerSec(kTargetBitrateBps),
3239 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003240
asaperssonfab67072017-04-04 05:51:49 -07003241 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003242 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003243
asaperssonfab67072017-04-04 05:51:49 -07003244 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003245 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003246
asaperssonfab67072017-04-04 05:51:49 -07003247 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003248 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003249
asaperssonfab67072017-04-04 05:51:49 -07003250 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003251 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003252
kthelgason876222f2016-11-29 01:44:11 -08003253 // Expect a scale down.
3254 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003255 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003256
asapersson02465b82017-04-10 01:12:52 -07003257 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003258 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003260 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003261
asaperssonfab67072017-04-04 05:51:49 -07003262 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003263 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003264 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003265 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003266
asaperssonfab67072017-04-04 05:51:49 -07003267 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003268 EXPECT_EQ(std::numeric_limits<int>::max(),
3269 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003270
asaperssonfab67072017-04-04 05:51:49 -07003271 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003272 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003273 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003274 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003275
asapersson02465b82017-04-10 01:12:52 -07003276 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003277 EXPECT_EQ(std::numeric_limits<int>::max(),
3278 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003279
mflodmancc3d4422017-08-03 08:27:51 -07003280 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003281}
3282
mflodmancc3d4422017-08-03 08:27:51 -07003283TEST_F(VideoStreamEncoderTest,
3284 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003285 const int kWidth = 1280;
3286 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003288 DataRate::BitsPerSec(kTargetBitrateBps),
3289 DataRate::BitsPerSec(kTargetBitrateBps),
3290 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003291
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003292 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003293 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003295 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003296
3297 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003298 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003299 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003300 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3301 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3302
3303 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003304 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003305 EXPECT_THAT(source.sink_wants(),
3306 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003307 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3308 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3310
3311 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003312 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003313 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3314 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3316
mflodmancc3d4422017-08-03 08:27:51 -07003317 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003318}
3319
mflodmancc3d4422017-08-03 08:27:51 -07003320TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003321 const int kWidth = 1280;
3322 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003323 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003324 DataRate::BitsPerSec(kTargetBitrateBps),
3325 DataRate::BitsPerSec(kTargetBitrateBps),
3326 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003327
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003328 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003329 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003330 video_stream_encoder_->SetSource(&source,
3331 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003332 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3333 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003334 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003335
3336 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003338 EXPECT_THAT(source.sink_wants(),
3339 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003340 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3341 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3342 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3343
3344 // Trigger adapt down for same input resolution, expect no change.
3345 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3346 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003347 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003348 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3349 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3350 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3351
3352 // Trigger adapt down for larger input resolution, expect no change.
3353 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3354 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003356 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3358 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3359
mflodmancc3d4422017-08-03 08:27:51 -07003360 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003361}
3362
mflodmancc3d4422017-08-03 08:27:51 -07003363TEST_F(VideoStreamEncoderTest,
3364 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003365 const int kWidth = 1280;
3366 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003367 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003368 DataRate::BitsPerSec(kTargetBitrateBps),
3369 DataRate::BitsPerSec(kTargetBitrateBps),
3370 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003371
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003372 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003373 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003374 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003375 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003376
3377 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003378 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003379 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003380 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3382
3383 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003384 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003386 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3388
mflodmancc3d4422017-08-03 08:27:51 -07003389 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003390}
3391
mflodmancc3d4422017-08-03 08:27:51 -07003392TEST_F(VideoStreamEncoderTest,
3393 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003394 const int kWidth = 1280;
3395 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003397 DataRate::BitsPerSec(kTargetBitrateBps),
3398 DataRate::BitsPerSec(kTargetBitrateBps),
3399 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003400
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003401 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003402 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003403 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003404 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003405
3406 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003407 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003408 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003409 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003410 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3411
3412 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003413 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003414 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003415 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003416 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3417
mflodmancc3d4422017-08-03 08:27:51 -07003418 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003419}
3420
mflodmancc3d4422017-08-03 08:27:51 -07003421TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003422 const int kWidth = 1280;
3423 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003424 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003425 DataRate::BitsPerSec(kTargetBitrateBps),
3426 DataRate::BitsPerSec(kTargetBitrateBps),
3427 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003428
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003429 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003430 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003431 video_stream_encoder_->SetSource(&source,
3432 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003433
3434 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3435 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003436 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003437 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3438 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3439 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3440
3441 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003442 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003443 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003444 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3445 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3446 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3447
mflodmancc3d4422017-08-03 08:27:51 -07003448 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003449}
3450
mflodmancc3d4422017-08-03 08:27:51 -07003451TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003452 const int kWidth = 1280;
3453 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003455 DataRate::BitsPerSec(kTargetBitrateBps),
3456 DataRate::BitsPerSec(kTargetBitrateBps),
3457 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003458
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003459 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003460 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003461 video_stream_encoder_->SetSource(&source,
3462 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003463
3464 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3465 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003466 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3469 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3470
3471 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003472 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003473 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003474 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3475 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477
mflodmancc3d4422017-08-03 08:27:51 -07003478 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003479}
3480
mflodmancc3d4422017-08-03 08:27:51 -07003481TEST_F(VideoStreamEncoderTest,
3482 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003483 const int kWidth = 1280;
3484 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003486 DataRate::BitsPerSec(kTargetBitrateBps),
3487 DataRate::BitsPerSec(kTargetBitrateBps),
3488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003489
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003490 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003491 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003492 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003493 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003494 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003495
3496 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003497 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3500 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3501
3502 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003503 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003504 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003505 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003506 EXPECT_THAT(source.sink_wants(),
3507 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3509 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3510
3511 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003512 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003513 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3515 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3517
mflodmancc3d4422017-08-03 08:27:51 -07003518 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003519}
3520
mflodmancc3d4422017-08-03 08:27:51 -07003521TEST_F(VideoStreamEncoderTest,
3522 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003523 const int kWidth = 1280;
3524 const int kHeight = 720;
3525 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003527 DataRate::BitsPerSec(kTargetBitrateBps),
3528 DataRate::BitsPerSec(kTargetBitrateBps),
3529 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003530
3531 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3532 stats.input_frame_rate = kInputFps;
3533 stats_proxy_->SetMockStats(stats);
3534
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003535 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003536 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3537 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003538 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003539
3540 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003541 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003542 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3543 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003544 EXPECT_THAT(video_source_.sink_wants(),
3545 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003546
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003547 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003548 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003549 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003550 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003551 // Give the encoder queue time to process the change in degradation preference
3552 // by waiting for an encoded frame.
3553 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3554 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003555 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003556
3557 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003558 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003559 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3560 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003561 EXPECT_THAT(new_video_source.sink_wants(),
3562 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003563
3564 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003565 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003566 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003567
mflodmancc3d4422017-08-03 08:27:51 -07003568 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003569}
3570
mflodmancc3d4422017-08-03 08:27:51 -07003571TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003572 const int kWidth = 1280;
3573 const int kHeight = 720;
3574 const size_t kNumFrames = 10;
3575
Henrik Boström381d1092020-05-12 18:49:07 +02003576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003577 DataRate::BitsPerSec(kTargetBitrateBps),
3578 DataRate::BitsPerSec(kTargetBitrateBps),
3579 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003580
asaperssond0de2952017-04-21 01:47:31 -07003581 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003582 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003583 video_source_.set_adaptation_enabled(true);
3584
3585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587
3588 int downscales = 0;
3589 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003590 video_source_.IncomingCapturedFrame(
3591 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3592 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003593
asaperssonfab67072017-04-04 05:51:49 -07003594 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003595 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003596 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003597 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003598
3599 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3600 ++downscales;
3601
3602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3603 EXPECT_EQ(downscales,
3604 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3605 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003606 }
mflodmancc3d4422017-08-03 08:27:51 -07003607 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003608}
3609
mflodmancc3d4422017-08-03 08:27:51 -07003610TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003611 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3612 const int kWidth = 1280;
3613 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003614 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003615 DataRate::BitsPerSec(kTargetBitrateBps),
3616 DataRate::BitsPerSec(kTargetBitrateBps),
3617 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003618
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003619 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003620 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003621 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003622 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003623 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003624
Åsa Persson8c1bf952018-09-13 10:42:19 +02003625 int64_t timestamp_ms = kFrameIntervalMs;
3626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003627 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003628 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3630 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3631
3632 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003633 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003634 timestamp_ms += kFrameIntervalMs;
3635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3636 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003637 EXPECT_THAT(source.sink_wants(),
3638 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003639 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3641
3642 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003643 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003644 timestamp_ms += kFrameIntervalMs;
3645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003646 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003647 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3649 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3650
3651 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003652 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003653 timestamp_ms += kFrameIntervalMs;
3654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003656 EXPECT_THAT(source.sink_wants(),
3657 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003658 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3659 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3660
3661 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003662 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003663 timestamp_ms += kFrameIntervalMs;
3664 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003665 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3668 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3669
mflodmancc3d4422017-08-03 08:27:51 -07003670 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003671}
3672
mflodmancc3d4422017-08-03 08:27:51 -07003673TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003674 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3675 const int kWidth = 1280;
3676 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003678 DataRate::BitsPerSec(kTargetBitrateBps),
3679 DataRate::BitsPerSec(kTargetBitrateBps),
3680 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003681
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003682 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003683 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003684 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003685 video_stream_encoder_->SetSource(&source,
3686 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003687
Åsa Persson8c1bf952018-09-13 10:42:19 +02003688 int64_t timestamp_ms = kFrameIntervalMs;
3689 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003690 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003691 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003692 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3693 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3694
3695 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003697 timestamp_ms += kFrameIntervalMs;
3698 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3699 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003700 EXPECT_THAT(source.sink_wants(),
3701 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003702 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3703 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3704
3705 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003706 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003707 timestamp_ms += kFrameIntervalMs;
3708 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003709 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003710 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003711 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3712 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3713
3714 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003715 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003716 timestamp_ms += kFrameIntervalMs;
3717 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3718 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003719 EXPECT_THAT(source.sink_wants(),
3720 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003721 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3722 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3723
3724 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003725 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003726 timestamp_ms += kFrameIntervalMs;
3727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003728 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003729 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3731 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3732
mflodmancc3d4422017-08-03 08:27:51 -07003733 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003734}
3735
Sergey Silkin41c650b2019-10-14 13:12:19 +02003736TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3737 fake_encoder_.SetResolutionBitrateLimits(
3738 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3739
Henrik Boström381d1092020-05-12 18:49:07 +02003740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003741 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3742 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3743 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3744 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003745
3746 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003747 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003748 source.set_adaptation_enabled(true);
3749 video_stream_encoder_->SetSource(
3750 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3751
3752 // Insert 720p frame.
3753 int64_t timestamp_ms = kFrameIntervalMs;
3754 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3755 WaitForEncodedFrame(1280, 720);
3756
3757 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003759 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3760 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3761 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3762 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003763 video_stream_encoder_->TriggerQualityLow();
3764
3765 // Insert 720p frame. It should be downscaled and encoded.
3766 timestamp_ms += kFrameIntervalMs;
3767 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3768 WaitForEncodedFrame(960, 540);
3769
3770 // Trigger adapt up. Higher resolution should not be requested duo to lack
3771 // of bitrate.
3772 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003773 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003774
3775 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003777 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3778 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3779 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3780 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003781
3782 // Trigger adapt up. Higher resolution should be requested.
3783 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003784 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003785
3786 video_stream_encoder_->Stop();
3787}
3788
3789TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3790 fake_encoder_.SetResolutionBitrateLimits(
3791 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3792
3793 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003795 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3796 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3797 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3798 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003799
3800 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003801 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003802 source.set_adaptation_enabled(true);
3803 video_stream_encoder_->SetSource(
3804 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3805
3806 // Insert 720p frame. It should be dropped and lower resolution should be
3807 // requested.
3808 int64_t timestamp_ms = kFrameIntervalMs;
3809 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3810 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003811 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003812
3813 // Insert 720p frame. It should be downscaled and encoded.
3814 timestamp_ms += kFrameIntervalMs;
3815 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3816 WaitForEncodedFrame(960, 540);
3817
3818 video_stream_encoder_->Stop();
3819}
3820
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003821class BalancedDegradationTest : public VideoStreamEncoderTest {
3822 protected:
3823 void SetupTest() {
3824 // Reset encoder for field trials to take effect.
3825 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003826 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003827
3828 // Enable BALANCED preference.
3829 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003830 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3831 }
3832
3833 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003835 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3836 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003837 }
3838
Åsa Persson45b176f2019-09-30 11:19:05 +02003839 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003840 timestamp_ms_ += kFrameIntervalMs;
3841 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003842 }
3843
3844 void InsertFrameAndWaitForEncoded() {
3845 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003846 sink_.WaitForEncodedFrame(timestamp_ms_);
3847 }
3848
3849 const int kWidth = 640; // pixels:640x360=230400
3850 const int kHeight = 360;
3851 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3852 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003853 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003854};
3855
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003856TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003857 test::ScopedFieldTrials field_trials(
3858 "WebRTC-Video-BalancedDegradationSettings/"
3859 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3860 SetupTest();
3861
3862 // Force input frame rate.
3863 const int kInputFps = 24;
3864 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3865 stats.input_frame_rate = kInputFps;
3866 stats_proxy_->SetMockStats(stats);
3867
Åsa Persson45b176f2019-09-30 11:19:05 +02003868 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003869 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003870
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003871 // Trigger adapt down, expect scaled down framerate and resolution,
3872 // since Fps diff (input-requested:0) < threshold.
3873 video_stream_encoder_->TriggerQualityLow();
3874 EXPECT_THAT(source_.sink_wants(),
3875 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003876
3877 video_stream_encoder_->Stop();
3878}
3879
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003880TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003881 test::ScopedFieldTrials field_trials(
3882 "WebRTC-Video-BalancedDegradationSettings/"
3883 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3884 SetupTest();
3885
3886 // Force input frame rate.
3887 const int kInputFps = 25;
3888 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3889 stats.input_frame_rate = kInputFps;
3890 stats_proxy_->SetMockStats(stats);
3891
Åsa Persson45b176f2019-09-30 11:19:05 +02003892 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003893 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003894
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003895 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3896 // Fps diff (input-requested:1) == threshold.
3897 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003898 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003899
3900 video_stream_encoder_->Stop();
3901}
3902
3903TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3904 test::ScopedFieldTrials field_trials(
3905 "WebRTC-Video-BalancedDegradationSettings/"
3906 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3907 SetupTest();
3908
3909 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3910
Åsa Persson45b176f2019-09-30 11:19:05 +02003911 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003912 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003913
3914 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3915 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003916 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003917
3918 video_stream_encoder_->Stop();
3919}
3920
Åsa Perssonccfb3402019-09-25 15:13:04 +02003921TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003922 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003923 "WebRTC-Video-BalancedDegradationSettings/"
3924 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003925 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003926
Åsa Persson1b247f12019-08-14 17:26:39 +02003927 const int kMinBitrateBps = 425000;
3928 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003929 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003930
Åsa Persson45b176f2019-09-30 11:19:05 +02003931 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003932 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003933 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3934
3935 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3936 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003937 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003939 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3940
3941 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3942 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003943 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003944 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003945 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3946
Åsa Persson30ab0152019-08-27 12:22:33 +02003947 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3948 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003949 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003950 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003951 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003952 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3953
3954 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003955 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003956 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003957 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003958
Åsa Persson30ab0152019-08-27 12:22:33 +02003959 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003960 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003961 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003962 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003963 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003964 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3965
3966 video_stream_encoder_->Stop();
3967}
3968
Åsa Perssonccfb3402019-09-25 15:13:04 +02003969TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003970 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3971 test::ScopedFieldTrials field_trials(
3972 "WebRTC-Video-BalancedDegradationSettings/"
3973 "pixels:57600|129600|230400,fps:7|24|24/");
3974 SetupTest();
3975 OnBitrateUpdated(kLowTargetBitrateBps);
3976
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003977 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003978
3979 // Insert frame, expect scaled down:
3980 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3981 InsertFrame();
3982 EXPECT_FALSE(WaitForFrame(1000));
3983 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3984 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3985
3986 // Insert frame, expect scaled down:
3987 // resolution (320x180@24fps).
3988 InsertFrame();
3989 EXPECT_FALSE(WaitForFrame(1000));
3990 EXPECT_LT(source_.sink_wants().max_pixel_count,
3991 source_.last_wants().max_pixel_count);
3992 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3993
3994 // Frame should not be dropped (min pixels per frame reached).
3995 InsertFrameAndWaitForEncoded();
3996
3997 video_stream_encoder_->Stop();
3998}
3999
4000TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004001 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004002 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004003 "WebRTC-Video-BalancedDegradationSettings/"
4004 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004005 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004006
Åsa Persson30ab0152019-08-27 12:22:33 +02004007 const int kResolutionMinBitrateBps = 435000;
4008 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004009 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004010
Åsa Persson45b176f2019-09-30 11:19:05 +02004011 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004012 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004013 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4014
4015 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4016 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004017 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004018 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004019 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4020
4021 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4022 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004023 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004024 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004025 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4026
4027 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4028 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004029 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004030 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004031 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4032
Åsa Persson30ab0152019-08-27 12:22:33 +02004033 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4034 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004035 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004036 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004037 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4038
4039 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4040 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004041 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004042 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4043
4044 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004045 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004046 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004047 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004048 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004049 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4050
4051 video_stream_encoder_->Stop();
4052}
4053
Åsa Perssonccfb3402019-09-25 15:13:04 +02004054TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004055 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004056 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004057 "WebRTC-Video-BalancedDegradationSettings/"
4058 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004059 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004060
Åsa Persson30ab0152019-08-27 12:22:33 +02004061 const int kMinBitrateBps = 425000;
4062 const int kTooLowMinBitrateBps = 424000;
4063 const int kResolutionMinBitrateBps = 435000;
4064 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004065 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004066
Åsa Persson45b176f2019-09-30 11:19:05 +02004067 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004068 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004069 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4070
4071 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4072 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004073 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004074 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004075 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4076
4077 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4078 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004079 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004080 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004081 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4082
4083 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4084 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004085 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004086 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004087 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4088
4089 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4090 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004091 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004092 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4093
4094 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004095 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004096 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004097 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004098 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004099 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4100
4101 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004102 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004103 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004104 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004105 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4106
4107 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004108 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004109 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004110 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004111 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004112 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4113
Åsa Persson1b247f12019-08-14 17:26:39 +02004114 video_stream_encoder_->Stop();
4115}
4116
mflodmancc3d4422017-08-03 08:27:51 -07004117TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004118 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4119 const int kWidth = 1280;
4120 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004122 DataRate::BitsPerSec(kTargetBitrateBps),
4123 DataRate::BitsPerSec(kTargetBitrateBps),
4124 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004125
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004126 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004127 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004128 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004129 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004130 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004131
Åsa Persson8c1bf952018-09-13 10:42:19 +02004132 int64_t timestamp_ms = kFrameIntervalMs;
4133 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004134 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004135 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004136 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4137 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4138 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4139 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4140
4141 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004142 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004143 timestamp_ms += kFrameIntervalMs;
4144 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4145 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004146 EXPECT_THAT(source.sink_wants(),
4147 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004148 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4149 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4150 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4151 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4152
4153 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004154 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004155 timestamp_ms += kFrameIntervalMs;
4156 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4157 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004158 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004159 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4160 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4161 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4162 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4163
Jonathan Yubc771b72017-12-08 17:04:29 -08004164 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004165 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004166 timestamp_ms += kFrameIntervalMs;
4167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4168 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004169 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004170 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4171 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004172 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004173 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4174
Jonathan Yubc771b72017-12-08 17:04:29 -08004175 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004176 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004177 timestamp_ms += kFrameIntervalMs;
4178 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4179 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004180 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004181 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004182 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4183 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4184 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4185 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4186
Jonathan Yubc771b72017-12-08 17:04:29 -08004187 // Trigger quality adapt down, expect no change (min resolution reached).
4188 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004189 timestamp_ms += kFrameIntervalMs;
4190 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4191 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004192 EXPECT_THAT(source.sink_wants(), FpsMax());
4193 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004194 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4195 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4196 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4197 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4198
Evan Shrubsole64469032020-06-11 10:45:29 +02004199 // Trigger quality adapt up, expect upscaled resolution (480x270).
4200 video_stream_encoder_->TriggerQualityHigh();
4201 timestamp_ms += kFrameIntervalMs;
4202 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4203 WaitForEncodedFrame(timestamp_ms);
4204 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4205 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4206 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4207 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4208 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4209
4210 // Trigger quality and cpu adapt up since both are most limited, expect
4211 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004212 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004213 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004214 timestamp_ms += kFrameIntervalMs;
4215 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4216 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004217 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004218 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4219 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4220 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004221 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004222
Evan Shrubsole64469032020-06-11 10:45:29 +02004223 // Trigger quality and cpu adapt up since both are most limited, expect
4224 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004225 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004226 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004227 timestamp_ms += kFrameIntervalMs;
4228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4229 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004230 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004231 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004232 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004233 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004234 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4235 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004236
Evan Shrubsole64469032020-06-11 10:45:29 +02004237 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4238 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004239 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004240 timestamp_ms += kFrameIntervalMs;
4241 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4242 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004243 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004244 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4245 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004246 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004247 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004248
4249 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004250 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004251 timestamp_ms += kFrameIntervalMs;
4252 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004253 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004254 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004256 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004258 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004259 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004260
mflodmancc3d4422017-08-03 08:27:51 -07004261 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004262}
4263
mflodmancc3d4422017-08-03 08:27:51 -07004264TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004265 const int kWidth = 640;
4266 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004267
Henrik Boström381d1092020-05-12 18:49:07 +02004268 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004269 DataRate::BitsPerSec(kTargetBitrateBps),
4270 DataRate::BitsPerSec(kTargetBitrateBps),
4271 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004272
perkj803d97f2016-11-01 11:45:46 -07004273 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004274 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004275 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004276 }
4277
mflodmancc3d4422017-08-03 08:27:51 -07004278 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004279 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004280 video_source_.IncomingCapturedFrame(CreateFrame(
4281 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004282 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004283 }
4284
mflodmancc3d4422017-08-03 08:27:51 -07004285 video_stream_encoder_->Stop();
4286 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004287 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004288
Ying Wangef3998f2019-12-09 13:06:53 +01004289 EXPECT_METRIC_EQ(
4290 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4291 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004292 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4293}
4294
mflodmancc3d4422017-08-03 08:27:51 -07004295TEST_F(VideoStreamEncoderTest,
4296 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004297 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004298 DataRate::BitsPerSec(kTargetBitrateBps),
4299 DataRate::BitsPerSec(kTargetBitrateBps),
4300 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004301 const int kWidth = 640;
4302 const int kHeight = 360;
4303
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004304 video_stream_encoder_->SetSource(&video_source_,
4305 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004306
4307 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4308 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004309 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004310 }
4311
mflodmancc3d4422017-08-03 08:27:51 -07004312 video_stream_encoder_->Stop();
4313 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004314 stats_proxy_.reset();
4315
4316 EXPECT_EQ(0,
4317 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4318}
4319
Per Kjellanderdcef6412020-10-07 15:09:05 +02004320TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4321 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004322 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004323 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004324
4325 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004326 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004327 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004328 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4329 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004330
Henrik Boström381d1092020-05-12 18:49:07 +02004331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004332 DataRate::BitsPerSec(kLowTargetBitrateBps),
4333 DataRate::BitsPerSec(kLowTargetBitrateBps),
4334 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004335
sprang57c2fff2017-01-16 06:24:02 -08004336 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004337 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4338 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004339 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4340 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4341
Erik Språngd7329ca2019-02-21 21:19:53 +01004342 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004343 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004344 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004345
Per Kjellanderdcef6412020-10-07 15:09:05 +02004346 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004347 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004348 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4349 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004350 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004351 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004352
Per Kjellanderdcef6412020-10-07 15:09:05 +02004353 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004354 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004355 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004356 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004357 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4358 WaitForEncodedFrame(CurrentTimeMs());
4359 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004360 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004361 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004362
mflodmancc3d4422017-08-03 08:27:51 -07004363 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004364}
4365
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004366TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004367 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004368 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004369 kVideoLayersAllocation);
4370
4371 const int kDefaultFps = 30;
4372
4373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4374 DataRate::BitsPerSec(kLowTargetBitrateBps),
4375 DataRate::BitsPerSec(kLowTargetBitrateBps),
4376 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4377
4378 video_source_.IncomingCapturedFrame(
4379 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4380 WaitForEncodedFrame(CurrentTimeMs());
4381 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4382 VideoLayersAllocation last_layer_allocation =
4383 sink_.GetLastVideoLayersAllocation();
4384 // kLowTargetBitrateBps is only enough for one spatial layer.
4385 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4386
4387 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004388 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004389 // Check that encoder has been updated too, not just allocation observer.
4390 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4391 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4392
Erik Språng9d69cbe2020-10-22 17:44:42 +02004393 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004394 int number_of_layers_allocation = 1;
4395 const int64_t start_time_ms = CurrentTimeMs();
4396 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4397 video_source_.IncomingCapturedFrame(
4398 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4399 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004400 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4401 number_of_layers_allocation = sink_.number_of_layers_allocations();
4402 VideoLayersAllocation new_allocation =
4403 sink_.GetLastVideoLayersAllocation();
4404 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4405 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4406 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4407 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4408 .target_bitrate_per_temporal_layer,
4409 last_layer_allocation.active_spatial_layers[0]
4410 .target_bitrate_per_temporal_layer);
4411 last_layer_allocation = new_allocation;
4412 }
4413 }
4414 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4415 video_stream_encoder_->Stop();
4416}
4417
4418TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004419 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004420 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4421 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4422 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004423 VideoEncoderConfig video_encoder_config;
4424 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4425 /* num_streams*/ 3, &video_encoder_config);
4426 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4427 video_encoder_config.content_type =
4428 VideoEncoderConfig::ContentType::kRealtimeVideo;
4429 video_encoder_config.encoder_specific_settings =
4430 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4431 VideoEncoder::GetDefaultVp8Settings());
4432 for (auto& layer : video_encoder_config.simulcast_layers) {
4433 layer.num_temporal_layers = 2;
4434 }
4435 // Simulcast layers are used for enabling/disabling streams.
4436 video_encoder_config.simulcast_layers[0].active = true;
4437 video_encoder_config.simulcast_layers[1].active = false;
4438 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004439 ConfigureEncoder(std::move(video_encoder_config),
4440 VideoStreamEncoder::BitrateAllocationCallbackType::
4441 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004442
4443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4444 DataRate::BitsPerSec(kTargetBitrateBps),
4445 DataRate::BitsPerSec(kTargetBitrateBps),
4446 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4447
4448 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4449 WaitForEncodedFrame(CurrentTimeMs());
4450 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4451 VideoLayersAllocation last_layer_allocation =
4452 sink_.GetLastVideoLayersAllocation();
4453
4454 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4455 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4456 .target_bitrate_per_temporal_layer,
4457 SizeIs(2));
4458 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4459 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4460 video_stream_encoder_->Stop();
4461}
4462
4463TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004464 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004465 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4466 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4467 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004468 VideoEncoderConfig video_encoder_config;
4469 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4470 /* num_streams*/ 3, &video_encoder_config);
4471 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4472 video_encoder_config.content_type =
4473 VideoEncoderConfig::ContentType::kRealtimeVideo;
4474 video_encoder_config.encoder_specific_settings =
4475 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4476 VideoEncoder::GetDefaultVp8Settings());
4477 for (auto& layer : video_encoder_config.simulcast_layers) {
4478 layer.num_temporal_layers = 2;
4479 }
4480 // Simulcast layers are used for enabling/disabling streams.
4481 video_encoder_config.simulcast_layers[0].active = true;
4482 video_encoder_config.simulcast_layers[1].active = false;
4483 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004484 ConfigureEncoder(std::move(video_encoder_config),
4485 VideoStreamEncoder::BitrateAllocationCallbackType::
4486 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004487
4488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4489 DataRate::BitsPerSec(kTargetBitrateBps),
4490 DataRate::BitsPerSec(kTargetBitrateBps),
4491 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4492
4493 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4494 WaitForEncodedFrame(CurrentTimeMs());
4495 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4496 VideoLayersAllocation last_layer_allocation =
4497 sink_.GetLastVideoLayersAllocation();
4498
4499 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4500 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4501 .target_bitrate_per_temporal_layer,
4502 SizeIs(2));
4503 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4504
4505 video_stream_encoder_->Stop();
4506}
4507
4508TEST_F(VideoStreamEncoderTest,
4509 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4510 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4511 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004512 VideoEncoderConfig video_encoder_config;
4513 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4514 /* num_streams*/ 1, &video_encoder_config);
4515 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4516 video_encoder_config.content_type =
4517 VideoEncoderConfig::ContentType::kRealtimeVideo;
4518 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4519 vp9_settings.numberOfSpatialLayers = 2;
4520 vp9_settings.numberOfTemporalLayers = 2;
4521 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4522 vp9_settings.automaticResizeOn = false;
4523 video_encoder_config.encoder_specific_settings =
4524 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4525 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004526 ConfigureEncoder(std::move(video_encoder_config),
4527 VideoStreamEncoder::BitrateAllocationCallbackType::
4528 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004529
4530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4531 DataRate::BitsPerSec(kTargetBitrateBps),
4532 DataRate::BitsPerSec(kTargetBitrateBps),
4533 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4534
4535 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4536 WaitForEncodedFrame(CurrentTimeMs());
4537 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4538 VideoLayersAllocation last_layer_allocation =
4539 sink_.GetLastVideoLayersAllocation();
4540
4541 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4542 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4543 .target_bitrate_per_temporal_layer,
4544 SizeIs(2));
4545 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4546 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4547 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4548 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4549 .target_bitrate_per_temporal_layer,
4550 SizeIs(2));
4551 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4552 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4553 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4554
4555 // Since full SVC is used, expect the top layer to utilize the full target
4556 // rate.
4557 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4558 .target_bitrate_per_temporal_layer[1],
4559 DataRate::BitsPerSec(kTargetBitrateBps));
4560 video_stream_encoder_->Stop();
4561}
4562
4563TEST_F(VideoStreamEncoderTest,
4564 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4565 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4566 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004567 VideoEncoderConfig video_encoder_config;
4568 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4569 /* num_streams*/ 1, &video_encoder_config);
4570 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4571 video_encoder_config.content_type =
4572 VideoEncoderConfig::ContentType::kRealtimeVideo;
4573 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4574 vp9_settings.numberOfSpatialLayers = 2;
4575 vp9_settings.numberOfTemporalLayers = 2;
4576 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4577 vp9_settings.automaticResizeOn = false;
4578 video_encoder_config.encoder_specific_settings =
4579 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4580 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004581 ConfigureEncoder(std::move(video_encoder_config),
4582 VideoStreamEncoder::BitrateAllocationCallbackType::
4583 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004584
4585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4586 DataRate::BitsPerSec(kTargetBitrateBps),
4587 DataRate::BitsPerSec(kTargetBitrateBps),
4588 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4589
4590 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4591 WaitForEncodedFrame(CurrentTimeMs());
4592 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4593 VideoLayersAllocation last_layer_allocation =
4594 sink_.GetLastVideoLayersAllocation();
4595
4596 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4597 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4598 .target_bitrate_per_temporal_layer,
4599 SizeIs(1));
4600 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4601 .target_bitrate_per_temporal_layer,
4602 SizeIs(1));
4603 // Since full SVC is used, expect the top layer to utilize the full target
4604 // rate.
4605 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4606 .target_bitrate_per_temporal_layer[0],
4607 DataRate::BitsPerSec(kTargetBitrateBps));
4608 video_stream_encoder_->Stop();
4609}
4610
4611TEST_F(VideoStreamEncoderTest,
4612 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4613 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4614 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004615 VideoEncoderConfig video_encoder_config;
4616 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4617 /* num_streams*/ 1, &video_encoder_config);
4618 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4619 video_encoder_config.content_type =
4620 VideoEncoderConfig::ContentType::kRealtimeVideo;
4621 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4622 vp9_settings.numberOfSpatialLayers = 2;
4623 vp9_settings.numberOfTemporalLayers = 2;
4624 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4625 vp9_settings.automaticResizeOn = false;
4626 video_encoder_config.encoder_specific_settings =
4627 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4628 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004629 ConfigureEncoder(std::move(video_encoder_config),
4630 VideoStreamEncoder::BitrateAllocationCallbackType::
4631 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004632
4633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4634 DataRate::BitsPerSec(kTargetBitrateBps),
4635 DataRate::BitsPerSec(kTargetBitrateBps),
4636 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4637
4638 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4639 WaitForEncodedFrame(CurrentTimeMs());
4640 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4641 VideoLayersAllocation last_layer_allocation =
4642 sink_.GetLastVideoLayersAllocation();
4643
4644 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4645 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4646 .target_bitrate_per_temporal_layer,
4647 SizeIs(2));
4648 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4649 .target_bitrate_per_temporal_layer,
4650 SizeIs(2));
4651 // Since KSVC is, spatial layers are independend except on key frames.
4652 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4653 .target_bitrate_per_temporal_layer[1],
4654 DataRate::BitsPerSec(kTargetBitrateBps));
4655 video_stream_encoder_->Stop();
4656}
4657
4658TEST_F(VideoStreamEncoderTest,
4659 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4660 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4661 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004663 VideoEncoderConfig video_encoder_config;
4664 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4665 /* num_streams*/ 1, &video_encoder_config);
4666 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4667 video_encoder_config.content_type =
4668 VideoEncoderConfig::ContentType::kRealtimeVideo;
4669 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4670 vp9_settings.numberOfSpatialLayers = 3;
4671 vp9_settings.numberOfTemporalLayers = 2;
4672 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4673 vp9_settings.automaticResizeOn = false;
4674 video_encoder_config.encoder_specific_settings =
4675 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4676 vp9_settings);
4677 // Simulcast layers are used for enabling/disabling streams.
4678 video_encoder_config.simulcast_layers.resize(3);
4679 video_encoder_config.simulcast_layers[0].active = false;
4680 video_encoder_config.simulcast_layers[1].active = true;
4681 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004682 ConfigureEncoder(std::move(video_encoder_config),
4683 VideoStreamEncoder::BitrateAllocationCallbackType::
4684 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004685
4686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4687 DataRate::BitsPerSec(kTargetBitrateBps),
4688 DataRate::BitsPerSec(kTargetBitrateBps),
4689 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4690
4691 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4692 WaitForEncodedFrame(CurrentTimeMs());
4693 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4694 VideoLayersAllocation last_layer_allocation =
4695 sink_.GetLastVideoLayersAllocation();
4696
4697 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4698 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4699 .target_bitrate_per_temporal_layer,
4700 SizeIs(2));
4701 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4702 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4703
4704 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4705 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4706 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4707 .target_bitrate_per_temporal_layer,
4708 SizeIs(2));
4709 // Since full SVC is used, expect the top layer to utilize the full target
4710 // rate.
4711 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4712 .target_bitrate_per_temporal_layer[1],
4713 DataRate::BitsPerSec(kTargetBitrateBps));
4714 video_stream_encoder_->Stop();
4715}
4716
4717TEST_F(VideoStreamEncoderTest,
4718 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4721 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004722 VideoEncoderConfig video_encoder_config;
4723 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4724 /* num_streams*/ 1, &video_encoder_config);
4725 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4726 video_encoder_config.content_type =
4727 VideoEncoderConfig::ContentType::kRealtimeVideo;
4728 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4729 vp9_settings.numberOfSpatialLayers = 3;
4730 vp9_settings.numberOfTemporalLayers = 2;
4731 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4732 vp9_settings.automaticResizeOn = false;
4733 video_encoder_config.encoder_specific_settings =
4734 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4735 vp9_settings);
4736 // Simulcast layers are used for enabling/disabling streams.
4737 video_encoder_config.simulcast_layers.resize(3);
4738 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004739 ConfigureEncoder(std::move(video_encoder_config),
4740 VideoStreamEncoder::BitrateAllocationCallbackType::
4741 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742
4743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4744 DataRate::BitsPerSec(kTargetBitrateBps),
4745 DataRate::BitsPerSec(kTargetBitrateBps),
4746 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4747
4748 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4749 WaitForEncodedFrame(CurrentTimeMs());
4750 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4751 VideoLayersAllocation last_layer_allocation =
4752 sink_.GetLastVideoLayersAllocation();
4753
4754 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4755 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4756 .target_bitrate_per_temporal_layer,
4757 SizeIs(2));
4758 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4759 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4760
4761 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4762 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4763 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4764 .target_bitrate_per_temporal_layer,
4765 SizeIs(2));
4766 video_stream_encoder_->Stop();
4767}
4768
4769TEST_F(VideoStreamEncoderTest,
4770 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4771 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4772 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4773 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004774 VideoEncoderConfig video_encoder_config;
4775 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4776 /* num_streams*/ 1, &video_encoder_config);
4777 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4778 video_encoder_config.content_type =
4779 VideoEncoderConfig::ContentType::kRealtimeVideo;
4780 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4781 vp9_settings.numberOfSpatialLayers = 3;
4782 vp9_settings.numberOfTemporalLayers = 2;
4783 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4784 vp9_settings.automaticResizeOn = false;
4785 video_encoder_config.encoder_specific_settings =
4786 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4787 vp9_settings);
4788 // Simulcast layers are used for enabling/disabling streams.
4789 video_encoder_config.simulcast_layers.resize(3);
4790 video_encoder_config.simulcast_layers[0].active = false;
4791 video_encoder_config.simulcast_layers[1].active = false;
4792 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004793 ConfigureEncoder(std::move(video_encoder_config),
4794 VideoStreamEncoder::BitrateAllocationCallbackType::
4795 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004796
4797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4798 DataRate::BitsPerSec(kTargetBitrateBps),
4799 DataRate::BitsPerSec(kTargetBitrateBps),
4800 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4801
4802 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4803 WaitForEncodedFrame(CurrentTimeMs());
4804 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4805 VideoLayersAllocation last_layer_allocation =
4806 sink_.GetLastVideoLayersAllocation();
4807
4808 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4809 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4810 .target_bitrate_per_temporal_layer,
4811 SizeIs(2));
4812 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4813 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4814 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4815 .target_bitrate_per_temporal_layer[1],
4816 DataRate::BitsPerSec(kTargetBitrateBps));
4817 video_stream_encoder_->Stop();
4818}
4819
4820TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4821 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004822 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823 kVideoLayersAllocation);
4824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4825 DataRate::BitsPerSec(kTargetBitrateBps),
4826 DataRate::BitsPerSec(kTargetBitrateBps),
4827 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4828
4829 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4830 WaitForEncodedFrame(CurrentTimeMs());
4831 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4832 VideoLayersAllocation last_layer_allocation =
4833 sink_.GetLastVideoLayersAllocation();
4834
4835 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4836 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4837 .target_bitrate_per_temporal_layer,
4838 SizeIs(1));
4839 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4840 .target_bitrate_per_temporal_layer[0],
4841 DataRate::BitsPerSec(kTargetBitrateBps));
4842 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4843 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4844 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4845 video_stream_encoder_->Stop();
4846}
4847
4848TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004849 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4850 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004851 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004852 kVideoLayersAllocation);
4853
4854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4855 DataRate::BitsPerSec(kLowTargetBitrateBps),
4856 DataRate::BitsPerSec(kLowTargetBitrateBps),
4857 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4858
4859 video_source_.IncomingCapturedFrame(
4860 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4861 WaitForEncodedFrame(CurrentTimeMs());
4862 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4863 VideoLayersAllocation last_layer_allocation =
4864 sink_.GetLastVideoLayersAllocation();
4865 // kLowTargetBitrateBps is only enough for one spatial layer.
4866 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4867 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4868 .target_bitrate_per_temporal_layer[0],
4869 DataRate::BitsPerSec(kLowTargetBitrateBps));
4870
4871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4872 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4873 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4874 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4875 video_source_.IncomingCapturedFrame(
4876 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4877 WaitForEncodedFrame(CurrentTimeMs());
4878
4879 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4880 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4881 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4882 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer[0],
4884 DataRate::Zero());
4885
4886 video_stream_encoder_->Stop();
4887}
4888
Per Kjellander4190ce92020-12-15 17:24:55 +01004889TEST_F(VideoStreamEncoderTest,
4890 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4891 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004892 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004893 kVideoLayersAllocation);
4894
4895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4896 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4897 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4898 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4899
4900 video_source_.IncomingCapturedFrame(
4901 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4902 WaitForEncodedFrame(CurrentTimeMs());
4903 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4904 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4905 SizeIs(2));
4906 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4907 codec_width_);
4908 EXPECT_EQ(
4909 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4910 codec_height_);
4911
4912 video_source_.IncomingCapturedFrame(
4913 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4914 WaitForEncodedFrame(CurrentTimeMs());
4915 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4916 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4917 SizeIs(2));
4918 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4919 codec_width_ / 2);
4920 EXPECT_EQ(
4921 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4922 codec_height_ / 2);
4923
4924 video_stream_encoder_->Stop();
4925}
4926
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004927TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4928 // 2 TLs configured, temporal layers supported by encoder.
4929 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004930 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004931 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004932 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004933 fake_encoder_.SetTemporalLayersSupported(0, true);
4934
4935 // Bitrate allocated across temporal layers.
4936 const int kTl0Bps = kTargetBitrateBps *
4937 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004938 kNumTemporalLayers, /*temporal_id*/ 0,
4939 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004940 const int kTl1Bps = kTargetBitrateBps *
4941 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004942 kNumTemporalLayers, /*temporal_id*/ 1,
4943 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004944 VideoBitrateAllocation expected_bitrate;
4945 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4946 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4947
4948 VerifyAllocatedBitrate(expected_bitrate);
4949 video_stream_encoder_->Stop();
4950}
4951
4952TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4953 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004954 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004955 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004956 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004957 fake_encoder_.SetTemporalLayersSupported(0, false);
4958
4959 // Temporal layers not supported by the encoder.
4960 // Total bitrate should be at ti:0.
4961 VideoBitrateAllocation expected_bitrate;
4962 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4963
4964 VerifyAllocatedBitrate(expected_bitrate);
4965 video_stream_encoder_->Stop();
4966}
4967
4968TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004969 webrtc::test::ScopedFieldTrials field_trials(
4970 "WebRTC-Video-QualityScalerSettings/"
4971 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4972 // Reset encoder for field trials to take effect.
4973 ConfigureEncoder(video_encoder_config_.Copy());
4974
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004975 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004976 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004977 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004978 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004979 fake_encoder_.SetTemporalLayersSupported(0, true);
4980 fake_encoder_.SetTemporalLayersSupported(1, false);
4981
4982 const int kS0Bps = 150000;
4983 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004984 kS0Bps *
4985 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4986 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004987 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004988 kS0Bps *
4989 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4990 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004991 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4992 // Temporal layers not supported by si:1.
4993 VideoBitrateAllocation expected_bitrate;
4994 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4995 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4996 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4997
4998 VerifyAllocatedBitrate(expected_bitrate);
4999 video_stream_encoder_->Stop();
5000}
5001
Niels Möller7dc26b72017-12-06 10:27:48 +01005002TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5003 const int kFrameWidth = 1280;
5004 const int kFrameHeight = 720;
5005 const int kFramerate = 24;
5006
Henrik Boström381d1092020-05-12 18:49:07 +02005007 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005008 DataRate::BitsPerSec(kTargetBitrateBps),
5009 DataRate::BitsPerSec(kTargetBitrateBps),
5010 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005011 test::FrameForwarder source;
5012 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005013 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005014
5015 // Insert a single frame, triggering initial configuration.
5016 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5017 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5018
5019 EXPECT_EQ(
5020 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5021 kDefaultFramerate);
5022
5023 // Trigger reconfigure encoder (without resetting the entire instance).
5024 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005025 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5026 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005027 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005028 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005029 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005030 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5031
5032 // Detector should be updated with fps limit from codec config.
5033 EXPECT_EQ(
5034 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5035 kFramerate);
5036
5037 // Trigger overuse, max framerate should be reduced.
5038 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5039 stats.input_frame_rate = kFramerate;
5040 stats_proxy_->SetMockStats(stats);
5041 video_stream_encoder_->TriggerCpuOveruse();
5042 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5043 int adapted_framerate =
5044 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5045 EXPECT_LT(adapted_framerate, kFramerate);
5046
5047 // Trigger underuse, max framerate should go back to codec configured fps.
5048 // Set extra low fps, to make sure it's actually reset, not just incremented.
5049 stats = stats_proxy_->GetStats();
5050 stats.input_frame_rate = adapted_framerate / 2;
5051 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005052 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005053 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5054 EXPECT_EQ(
5055 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5056 kFramerate);
5057
5058 video_stream_encoder_->Stop();
5059}
5060
5061TEST_F(VideoStreamEncoderTest,
5062 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5063 const int kFrameWidth = 1280;
5064 const int kFrameHeight = 720;
5065 const int kLowFramerate = 15;
5066 const int kHighFramerate = 25;
5067
Henrik Boström381d1092020-05-12 18:49:07 +02005068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005069 DataRate::BitsPerSec(kTargetBitrateBps),
5070 DataRate::BitsPerSec(kTargetBitrateBps),
5071 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005072 test::FrameForwarder source;
5073 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005074 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005075
5076 // Trigger initial configuration.
5077 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005078 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5079 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005080 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005081 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005082 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005083 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005084 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5085
5086 EXPECT_EQ(
5087 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5088 kLowFramerate);
5089
5090 // Trigger overuse, max framerate should be reduced.
5091 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5092 stats.input_frame_rate = kLowFramerate;
5093 stats_proxy_->SetMockStats(stats);
5094 video_stream_encoder_->TriggerCpuOveruse();
5095 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5096 int adapted_framerate =
5097 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5098 EXPECT_LT(adapted_framerate, kLowFramerate);
5099
5100 // Reconfigure the encoder with a new (higher max framerate), max fps should
5101 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005102 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005103 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5104 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005105 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005106 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5107
5108 EXPECT_EQ(
5109 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5110 adapted_framerate);
5111
5112 // Trigger underuse, max framerate should go back to codec configured fps.
5113 stats = stats_proxy_->GetStats();
5114 stats.input_frame_rate = adapted_framerate;
5115 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005116 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005117 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5118 EXPECT_EQ(
5119 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5120 kHighFramerate);
5121
5122 video_stream_encoder_->Stop();
5123}
5124
mflodmancc3d4422017-08-03 08:27:51 -07005125TEST_F(VideoStreamEncoderTest,
5126 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005127 const int kFrameWidth = 1280;
5128 const int kFrameHeight = 720;
5129 const int kFramerate = 24;
5130
Henrik Boström381d1092020-05-12 18:49:07 +02005131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005132 DataRate::BitsPerSec(kTargetBitrateBps),
5133 DataRate::BitsPerSec(kTargetBitrateBps),
5134 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005135 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005136 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005137 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005138
5139 // Trigger initial configuration.
5140 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005141 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5142 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005143 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005144 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005145 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005146 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005147 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005148
Niels Möller7dc26b72017-12-06 10:27:48 +01005149 EXPECT_EQ(
5150 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5151 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005152
5153 // Trigger overuse, max framerate should be reduced.
5154 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5155 stats.input_frame_rate = kFramerate;
5156 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005157 video_stream_encoder_->TriggerCpuOveruse();
5158 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005159 int adapted_framerate =
5160 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005161 EXPECT_LT(adapted_framerate, kFramerate);
5162
5163 // Change degradation preference to not enable framerate scaling. Target
5164 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005165 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005166 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005167 EXPECT_EQ(
5168 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5169 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005170
mflodmancc3d4422017-08-03 08:27:51 -07005171 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005172}
5173
mflodmancc3d4422017-08-03 08:27:51 -07005174TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005175 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005177 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5178 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5179 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005180 const int kWidth = 640;
5181 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005182
asaperssonfab67072017-04-04 05:51:49 -07005183 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005184
5185 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005186 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005187
5188 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005189 EXPECT_TRUE_WAIT(
5190 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005191
sprangc5d62e22017-04-02 23:53:04 -07005192 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005193
asaperssonfab67072017-04-04 05:51:49 -07005194 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005195 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005196 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005197
5198 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005199 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005200
Henrik Boström2671dac2020-05-19 16:29:09 +02005201 EXPECT_TRUE_WAIT(
5202 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005203
mflodmancc3d4422017-08-03 08:27:51 -07005204 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005205}
5206
mflodmancc3d4422017-08-03 08:27:51 -07005207TEST_F(VideoStreamEncoderTest,
5208 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005209 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005211 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5212 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5213 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005214 const int kWidth = 640;
5215 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005216
5217 // We expect the n initial frames to get dropped.
5218 int i;
5219 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005220 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005221 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005222 }
5223 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005224 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005225 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005226
5227 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005228 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005229
mflodmancc3d4422017-08-03 08:27:51 -07005230 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005231}
5232
mflodmancc3d4422017-08-03 08:27:51 -07005233TEST_F(VideoStreamEncoderTest,
5234 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005235 const int kWidth = 640;
5236 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005238 DataRate::BitsPerSec(kLowTargetBitrateBps),
5239 DataRate::BitsPerSec(kLowTargetBitrateBps),
5240 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005241
5242 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005243 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005244 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005245
asaperssonfab67072017-04-04 05:51:49 -07005246 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005247 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005248 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005249
mflodmancc3d4422017-08-03 08:27:51 -07005250 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005251}
5252
mflodmancc3d4422017-08-03 08:27:51 -07005253TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005254 const int kWidth = 640;
5255 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005256 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005257
5258 VideoEncoderConfig video_encoder_config;
5259 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5260 // Make format different, to force recreation of encoder.
5261 video_encoder_config.video_format.parameters["foo"] = "foo";
5262 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005263 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005265 DataRate::BitsPerSec(kLowTargetBitrateBps),
5266 DataRate::BitsPerSec(kLowTargetBitrateBps),
5267 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005268
kthelgasonb83797b2017-02-14 11:57:25 -08005269 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005270 video_stream_encoder_->SetSource(&video_source_,
5271 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005272
asaperssonfab67072017-04-04 05:51:49 -07005273 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005274 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005275 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005276
mflodmancc3d4422017-08-03 08:27:51 -07005277 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005278 fake_encoder_.SetQualityScaling(true);
5279}
5280
Åsa Persson139f4dc2019-08-02 09:29:58 +02005281TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5282 webrtc::test::ScopedFieldTrials field_trials(
5283 "WebRTC-Video-QualityScalerSettings/"
5284 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5285 // Reset encoder for field trials to take effect.
5286 ConfigureEncoder(video_encoder_config_.Copy());
5287 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5288 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5289 const int kWidth = 640;
5290 const int kHeight = 360;
5291
Henrik Boström381d1092020-05-12 18:49:07 +02005292 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005293 DataRate::BitsPerSec(kTargetBitrateBps),
5294 DataRate::BitsPerSec(kTargetBitrateBps),
5295 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005296 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5297 // Frame should not be dropped.
5298 WaitForEncodedFrame(1);
5299
Henrik Boström381d1092020-05-12 18:49:07 +02005300 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005301 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5302 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5303 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005304 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5305 // Frame should not be dropped.
5306 WaitForEncodedFrame(2);
5307
Henrik Boström381d1092020-05-12 18:49:07 +02005308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005309 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5310 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5311 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005312 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5313 // Expect to drop this frame, the wait should time out.
5314 ExpectDroppedFrame();
5315
5316 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005317 EXPECT_TRUE_WAIT(
5318 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005319 video_stream_encoder_->Stop();
5320}
5321
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005322TEST_F(VideoStreamEncoderTest,
5323 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5324 webrtc::test::ScopedFieldTrials field_trials(
5325 "WebRTC-Video-QualityScalerSettings/"
5326 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5327 fake_encoder_.SetQualityScaling(false);
5328 ConfigureEncoder(video_encoder_config_.Copy());
5329 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5330 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5331 const int kWidth = 640;
5332 const int kHeight = 360;
5333
5334 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5335 DataRate::BitsPerSec(kTargetBitrateBps),
5336 DataRate::BitsPerSec(kTargetBitrateBps),
5337 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5338 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5339 // Frame should not be dropped.
5340 WaitForEncodedFrame(1);
5341
5342 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5343 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5344 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5345 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5346 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5347 // Frame should not be dropped.
5348 WaitForEncodedFrame(2);
5349
5350 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5351 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5352 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5353 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5354 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5355 // Not dropped since quality scaling is disabled.
5356 WaitForEncodedFrame(3);
5357
5358 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005360 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5361
5362 video_stream_encoder_->Stop();
5363}
5364
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005365TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5366 const int kLowTargetBitrateBps = 400000;
5367 // Set simulcast.
5368 ResetEncoder("VP8", 3, 1, 1, false);
5369 fake_encoder_.SetQualityScaling(true);
5370 const int kWidth = 1280;
5371 const int kHeight = 720;
5372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5373 DataRate::BitsPerSec(kLowTargetBitrateBps),
5374 DataRate::BitsPerSec(kLowTargetBitrateBps),
5375 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5376 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5377 // Frame should not be dropped.
5378 WaitForEncodedFrame(1);
5379
5380 // Trigger QVGA "singlecast"
5381 // Update the config.
5382 VideoEncoderConfig video_encoder_config;
5383 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5384 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005385 video_encoder_config.video_stream_factory =
5386 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5387 "VP8", /*max qp*/ 56, /*screencast*/ false,
5388 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005389 for (auto& layer : video_encoder_config.simulcast_layers) {
5390 layer.num_temporal_layers = 1;
5391 layer.max_framerate = kDefaultFramerate;
5392 }
5393 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5394 video_encoder_config.content_type =
5395 VideoEncoderConfig::ContentType::kRealtimeVideo;
5396
5397 video_encoder_config.simulcast_layers[0].active = true;
5398 video_encoder_config.simulcast_layers[1].active = false;
5399 video_encoder_config.simulcast_layers[2].active = false;
5400
5401 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5402 kMaxPayloadLength);
5403 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5404
5405 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5406 // Frame should not be dropped.
5407 WaitForEncodedFrame(2);
5408
5409 // Trigger HD "singlecast"
5410 video_encoder_config.simulcast_layers[0].active = false;
5411 video_encoder_config.simulcast_layers[1].active = false;
5412 video_encoder_config.simulcast_layers[2].active = true;
5413
5414 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5415 kMaxPayloadLength);
5416 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5417
5418 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5419 // Frame should be dropped because of initial frame drop.
5420 ExpectDroppedFrame();
5421
5422 // Expect the sink_wants to specify a scaled frame.
5423 EXPECT_TRUE_WAIT(
5424 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5425 video_stream_encoder_->Stop();
5426}
5427
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005428TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5429 const int kLowTargetBitrateBps = 400000;
5430 // Set simulcast.
5431 ResetEncoder("VP9", 1, 1, 3, false);
5432 fake_encoder_.SetQualityScaling(true);
5433 const int kWidth = 1280;
5434 const int kHeight = 720;
5435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5436 DataRate::BitsPerSec(kLowTargetBitrateBps),
5437 DataRate::BitsPerSec(kLowTargetBitrateBps),
5438 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5439 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5440 // Frame should not be dropped.
5441 WaitForEncodedFrame(1);
5442
5443 // Trigger QVGA "singlecast"
5444 // Update the config.
5445 VideoEncoderConfig video_encoder_config;
5446 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5447 &video_encoder_config);
5448 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5449 vp9_settings.numberOfSpatialLayers = 3;
5450 // Since only one layer is active - automatic resize should be enabled.
5451 vp9_settings.automaticResizeOn = true;
5452 video_encoder_config.encoder_specific_settings =
5453 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5454 vp9_settings);
5455 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5456 video_encoder_config.content_type =
5457 VideoEncoderConfig::ContentType::kRealtimeVideo;
5458 // Currently simulcast layers |active| flags are used to inidicate
5459 // which SVC layers are active.
5460 video_encoder_config.simulcast_layers.resize(3);
5461
5462 video_encoder_config.simulcast_layers[0].active = true;
5463 video_encoder_config.simulcast_layers[1].active = false;
5464 video_encoder_config.simulcast_layers[2].active = false;
5465
5466 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5467 kMaxPayloadLength);
5468 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5469
5470 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5471 // Frame should not be dropped.
5472 WaitForEncodedFrame(2);
5473
5474 // Trigger HD "singlecast"
5475 video_encoder_config.simulcast_layers[0].active = false;
5476 video_encoder_config.simulcast_layers[1].active = false;
5477 video_encoder_config.simulcast_layers[2].active = true;
5478
5479 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5480 kMaxPayloadLength);
5481 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5482
5483 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5484 // Frame should be dropped because of initial frame drop.
5485 ExpectDroppedFrame();
5486
5487 // Expect the sink_wants to specify a scaled frame.
5488 EXPECT_TRUE_WAIT(
5489 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5490 video_stream_encoder_->Stop();
5491}
5492
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005493TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005494 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5495 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5496 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5497 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5498 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5499 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5500 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5501 fake_encoder_.SetResolutionBitrateLimits(
5502 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5503
5504 VideoEncoderConfig video_encoder_config;
5505 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5506 &video_encoder_config);
5507 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5508 vp9_settings.numberOfSpatialLayers = 3;
5509 // Since only one layer is active - automatic resize should be enabled.
5510 vp9_settings.automaticResizeOn = true;
5511 video_encoder_config.encoder_specific_settings =
5512 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5513 vp9_settings);
5514 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5515 video_encoder_config.content_type =
5516 VideoEncoderConfig::ContentType::kRealtimeVideo;
5517 // Simulcast layers are used to indicate which spatial layers are active.
5518 video_encoder_config.simulcast_layers.resize(3);
5519 video_encoder_config.simulcast_layers[0].active = false;
5520 video_encoder_config.simulcast_layers[1].active = true;
5521 video_encoder_config.simulcast_layers[2].active = false;
5522
5523 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5524 kMaxPayloadLength);
5525 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5526
5527 // The encoder bitrate limits for 360p should be used.
5528 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5529 EXPECT_FALSE(WaitForFrame(1000));
5530 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5531 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5532 VideoCodecType::kVideoCodecVP9);
5533 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5534 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5535 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5536 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5537 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5538 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5539 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5540 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5541
5542 // The encoder bitrate limits for 270p should be used.
5543 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5544 EXPECT_FALSE(WaitForFrame(1000));
5545 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5546 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5547 VideoCodecType::kVideoCodecVP9);
5548 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5549 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5550 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5551 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5552 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5553 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5554 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5555 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5556
5557 video_stream_encoder_->Stop();
5558}
5559
5560TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005561 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5562 VideoEncoderConfig video_encoder_config;
5563 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5564 &video_encoder_config);
5565 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5566 vp9_settings.numberOfSpatialLayers = 3;
5567 // Since only one layer is active - automatic resize should be enabled.
5568 vp9_settings.automaticResizeOn = true;
5569 video_encoder_config.encoder_specific_settings =
5570 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5571 vp9_settings);
5572 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5573 video_encoder_config.content_type =
5574 VideoEncoderConfig::ContentType::kRealtimeVideo;
5575 // Simulcast layers are used to indicate which spatial layers are active.
5576 video_encoder_config.simulcast_layers.resize(3);
5577 video_encoder_config.simulcast_layers[0].active = false;
5578 video_encoder_config.simulcast_layers[1].active = true;
5579 video_encoder_config.simulcast_layers[2].active = false;
5580
5581 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5582 kMaxPayloadLength);
5583 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5584
5585 // The default bitrate limits for 360p should be used.
5586 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005587 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5588 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005589 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5590 EXPECT_FALSE(WaitForFrame(1000));
5591 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5592 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5593 VideoCodecType::kVideoCodecVP9);
5594 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5595 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5596 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5597 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5598 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5599 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5600 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5601 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5602
5603 // The default bitrate limits for 270p should be used.
5604 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005605 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5606 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005607 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5608 EXPECT_FALSE(WaitForFrame(1000));
5609 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5610 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5611 VideoCodecType::kVideoCodecVP9);
5612 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5613 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5614 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5615 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5616 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5617 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5618 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5619 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5620
5621 video_stream_encoder_->Stop();
5622}
5623
5624TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5625 webrtc::test::ScopedFieldTrials field_trials(
5626 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5627 VideoEncoderConfig video_encoder_config;
5628 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5629 &video_encoder_config);
5630 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5631 vp9_settings.numberOfSpatialLayers = 3;
5632 // Since only one layer is active - automatic resize should be enabled.
5633 vp9_settings.automaticResizeOn = true;
5634 video_encoder_config.encoder_specific_settings =
5635 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5636 vp9_settings);
5637 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5638 video_encoder_config.content_type =
5639 VideoEncoderConfig::ContentType::kRealtimeVideo;
5640 // Simulcast layers are used to indicate which spatial layers are active.
5641 video_encoder_config.simulcast_layers.resize(3);
5642 video_encoder_config.simulcast_layers[0].active = false;
5643 video_encoder_config.simulcast_layers[1].active = true;
5644 video_encoder_config.simulcast_layers[2].active = false;
5645
5646 // Reset encoder for field trials to take effect.
5647 ConfigureEncoder(video_encoder_config.Copy());
5648
5649 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5650 kMaxPayloadLength);
5651 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5652
5653 // The default bitrate limits for 360p should not be used.
5654 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005655 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5656 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005657 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5658 EXPECT_FALSE(WaitForFrame(1000));
5659 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005660 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005661 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5662 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5663 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5664 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5665 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5666 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5667
5668 video_stream_encoder_->Stop();
5669}
5670
5671TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5672 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5673 /*num_spatial_layers=*/1, /*screenshare=*/false);
5674
5675 // The default singlecast bitrate limits for 720p should not be used.
5676 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005677 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5678 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005679 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5680 EXPECT_FALSE(WaitForFrame(1000));
5681 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5682 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5683 VideoCodecType::kVideoCodecVP9);
5684 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5685 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5686 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5687 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5688 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5689 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5690
5691 video_stream_encoder_->Stop();
5692}
5693
5694TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005695 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5696 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5697 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5698 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5699 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5700 fake_encoder_.SetResolutionBitrateLimits(
5701 {kEncoderLimits180p, kEncoderLimits720p});
5702
5703 VideoEncoderConfig video_encoder_config;
5704 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5705 &video_encoder_config);
5706 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5707 vp9_settings.numberOfSpatialLayers = 3;
5708 // Since only one layer is active - automatic resize should be enabled.
5709 vp9_settings.automaticResizeOn = true;
5710 video_encoder_config.encoder_specific_settings =
5711 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5712 vp9_settings);
5713 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5714 video_encoder_config.content_type =
5715 VideoEncoderConfig::ContentType::kRealtimeVideo;
5716 // Simulcast layers are used to indicate which spatial layers are active.
5717 video_encoder_config.simulcast_layers.resize(3);
5718 video_encoder_config.simulcast_layers[0].active = true;
5719 video_encoder_config.simulcast_layers[1].active = false;
5720 video_encoder_config.simulcast_layers[2].active = false;
5721
5722 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5723 kMaxPayloadLength);
5724 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5725
5726 // Limits not applied on lowest stream, limits for 180p should not be used.
5727 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5728 EXPECT_FALSE(WaitForFrame(1000));
5729 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5730 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5731 VideoCodecType::kVideoCodecVP9);
5732 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5733 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5734 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5735 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5736 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5737 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5738 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5739 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5740
5741 video_stream_encoder_->Stop();
5742}
5743
5744TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005745 InitialFrameDropActivatesWhenResolutionIncreases) {
5746 const int kWidth = 640;
5747 const int kHeight = 360;
5748
5749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5750 DataRate::BitsPerSec(kTargetBitrateBps),
5751 DataRate::BitsPerSec(kTargetBitrateBps),
5752 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5753 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5754 // Frame should not be dropped.
5755 WaitForEncodedFrame(1);
5756
5757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5758 DataRate::BitsPerSec(kLowTargetBitrateBps),
5759 DataRate::BitsPerSec(kLowTargetBitrateBps),
5760 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5761 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5762 // Frame should not be dropped, bitrate not too low for frame.
5763 WaitForEncodedFrame(2);
5764
5765 // Incoming resolution increases.
5766 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5767 // Expect to drop this frame, bitrate too low for frame.
5768 ExpectDroppedFrame();
5769
5770 // Expect the sink_wants to specify a scaled frame.
5771 EXPECT_TRUE_WAIT(
5772 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5773 video_stream_encoder_->Stop();
5774}
5775
5776TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5777 const int kWidth = 640;
5778 const int kHeight = 360;
5779 // So that quality scaling doesn't happen by itself.
5780 fake_encoder_.SetQp(kQpHigh);
5781
5782 AdaptingFrameForwarder source(&time_controller_);
5783 source.set_adaptation_enabled(true);
5784 video_stream_encoder_->SetSource(
5785 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5786
5787 int timestamp = 1;
5788
5789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5790 DataRate::BitsPerSec(kTargetBitrateBps),
5791 DataRate::BitsPerSec(kTargetBitrateBps),
5792 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5793 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5794 WaitForEncodedFrame(timestamp);
5795 timestamp += 9000;
5796 // Long pause to disable all first BWE drop logic.
5797 AdvanceTime(TimeDelta::Millis(1000));
5798
5799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5800 DataRate::BitsPerSec(kLowTargetBitrateBps),
5801 DataRate::BitsPerSec(kLowTargetBitrateBps),
5802 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5803 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5804 // Not dropped frame, as initial frame drop is disabled by now.
5805 WaitForEncodedFrame(timestamp);
5806 timestamp += 9000;
5807 AdvanceTime(TimeDelta::Millis(100));
5808
5809 // Quality adaptation down.
5810 video_stream_encoder_->TriggerQualityLow();
5811
5812 // Adaptation has an effect.
5813 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5814 5000);
5815
5816 // Frame isn't dropped as initial frame dropper is disabled.
5817 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5818 WaitForEncodedFrame(timestamp);
5819 timestamp += 9000;
5820 AdvanceTime(TimeDelta::Millis(100));
5821
5822 // Quality adaptation up.
5823 video_stream_encoder_->TriggerQualityHigh();
5824
5825 // Adaptation has an effect.
5826 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5827 5000);
5828
5829 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5830 // Frame should not be dropped, as initial framedropper is off.
5831 WaitForEncodedFrame(timestamp);
5832
5833 video_stream_encoder_->Stop();
5834}
5835
Åsa Persson7f354f82021-02-04 15:52:15 +01005836TEST_F(VideoStreamEncoderTest,
5837 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5838 const int kMinStartBps360p = 222000;
5839 fake_encoder_.SetResolutionBitrateLimits(
5840 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5841 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5842 800000)});
5843
5844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5845 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5846 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5847 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5848 0, 0, 0);
5849 // Frame should not be dropped, bitrate not too low for frame.
5850 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5851 WaitForEncodedFrame(1);
5852
5853 // Incoming resolution increases, initial frame drop activates.
5854 // Frame should be dropped, link allocation too low for frame.
5855 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5856 ExpectDroppedFrame();
5857
5858 // Expect sink_wants to specify a scaled frame.
5859 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5860 5000);
5861 video_stream_encoder_->Stop();
5862}
5863
5864TEST_F(VideoStreamEncoderTest,
5865 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5866 const int kMinStartBps360p = 222000;
5867 fake_encoder_.SetResolutionBitrateLimits(
5868 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5869 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5870 800000)});
5871
5872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5873 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5874 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5875 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5876 0, 0, 0);
5877 // Frame should not be dropped, bitrate not too low for frame.
5878 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5879 WaitForEncodedFrame(1);
5880
5881 // Incoming resolution increases, initial frame drop activates.
5882 // Frame should be dropped, link allocation not too low for frame.
5883 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5884 WaitForEncodedFrame(2);
5885
5886 video_stream_encoder_->Stop();
5887}
5888
Åsa Perssone644a032019-11-08 15:56:00 +01005889TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5890 webrtc::test::ScopedFieldTrials field_trials(
5891 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5892
5893 // Reset encoder for field trials to take effect.
5894 VideoEncoderConfig config = video_encoder_config_.Copy();
5895 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005896 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005897 ConfigureEncoder(std::move(config));
5898 fake_encoder_.SetQp(kQpLow);
5899
5900 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005901 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005902 source.set_adaptation_enabled(true);
5903 video_stream_encoder_->SetSource(&source,
5904 DegradationPreference::MAINTAIN_FRAMERATE);
5905
5906 // Start at low bitrate.
5907 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5909 DataRate::BitsPerSec(kLowBitrateBps),
5910 DataRate::BitsPerSec(kLowBitrateBps),
5911 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005912
5913 // Expect first frame to be dropped and resolution to be limited.
5914 const int kWidth = 1280;
5915 const int kHeight = 720;
5916 const int64_t kFrameIntervalMs = 100;
5917 int64_t timestamp_ms = kFrameIntervalMs;
5918 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5919 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005920 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5921 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005922
5923 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5925 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005926
5927 // Insert frames and advance |min_duration_ms|.
5928 for (size_t i = 1; i <= 10; i++) {
5929 timestamp_ms += kFrameIntervalMs;
5930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5931 WaitForEncodedFrame(timestamp_ms);
5932 }
5933 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5934 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5935
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005936 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005937
5938 // Insert frame should trigger high BW and release quality limitation.
5939 timestamp_ms += kFrameIntervalMs;
5940 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5941 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005942 // The ramp-up code involves the adaptation queue, give it time to execute.
5943 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005944 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005945 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005946
5947 // Frame should not be adapted.
5948 timestamp_ms += kFrameIntervalMs;
5949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5950 WaitForEncodedFrame(kWidth, kHeight);
5951 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5952
5953 video_stream_encoder_->Stop();
5954}
5955
mflodmancc3d4422017-08-03 08:27:51 -07005956TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005957 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005958 webrtc::test::ScopedFieldTrials field_trials(
5959 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005960 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005961 source.set_adaptation_enabled(true);
5962 video_stream_encoder_->SetSource(&source,
5963 DegradationPreference::MAINTAIN_FRAMERATE);
5964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5965 DataRate::BitsPerSec(kTargetBitrateBps),
5966 DataRate::BitsPerSec(kTargetBitrateBps),
5967 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5968 fake_encoder_.SetQp(kQpHigh + 1);
5969 const int kWidth = 1280;
5970 const int kHeight = 720;
5971 const int64_t kFrameIntervalMs = 100;
5972 int64_t timestamp_ms = kFrameIntervalMs;
5973 for (size_t i = 1; i <= 100; i++) {
5974 timestamp_ms += kFrameIntervalMs;
5975 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5976 WaitForEncodedFrame(timestamp_ms);
5977 }
5978 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5979 // for the first time.
5980 // TODO(eshr): We should avoid these waits by using threads with simulated
5981 // time.
5982 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5983 2000 * 2.5 * 2);
5984 timestamp_ms += kFrameIntervalMs;
5985 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5986 WaitForEncodedFrame(timestamp_ms);
5987 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5988 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5989 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5990
5991 // Disable Quality scaling by turning off scaler on the encoder and
5992 // reconfiguring.
5993 fake_encoder_.SetQualityScaling(false);
5994 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5995 kMaxPayloadLength);
5996 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005997 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005998 // Since we turned off the quality scaler, the adaptations made by it are
5999 // removed.
6000 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6002
6003 video_stream_encoder_->Stop();
6004}
6005
6006TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006007 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6008 const int kTooSmallWidth = 10;
6009 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006011 DataRate::BitsPerSec(kTargetBitrateBps),
6012 DataRate::BitsPerSec(kTargetBitrateBps),
6013 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006014
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006015 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006016 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006017 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006018 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006019 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006020 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6021
6022 // Trigger adapt down, too small frame, expect no change.
6023 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006024 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006025 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006026 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006027 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6028 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6029
mflodmancc3d4422017-08-03 08:27:51 -07006030 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006031}
6032
mflodmancc3d4422017-08-03 08:27:51 -07006033TEST_F(VideoStreamEncoderTest,
6034 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006035 const int kTooSmallWidth = 10;
6036 const int kTooSmallHeight = 10;
6037 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006039 DataRate::BitsPerSec(kTargetBitrateBps),
6040 DataRate::BitsPerSec(kTargetBitrateBps),
6041 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006042
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006043 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006044 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006045 video_stream_encoder_->SetSource(&source,
6046 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006047 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006048 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6049 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6050
6051 // Trigger adapt down, expect limited framerate.
6052 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006053 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006054 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006055 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006056 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6057 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6058 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6059
6060 // Trigger adapt down, too small frame, expect no change.
6061 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006062 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006063 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006064 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006065 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6066 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6067 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6068
mflodmancc3d4422017-08-03 08:27:51 -07006069 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006070}
6071
mflodmancc3d4422017-08-03 08:27:51 -07006072TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006073 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006075 DataRate::BitsPerSec(kTargetBitrateBps),
6076 DataRate::BitsPerSec(kTargetBitrateBps),
6077 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006078 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006079 const int kFrameWidth = 1280;
6080 const int kFrameHeight = 720;
6081 video_source_.IncomingCapturedFrame(
6082 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006083 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006084 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006085}
6086
sprangb1ca0732017-02-01 08:38:12 -08006087// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006088TEST_F(VideoStreamEncoderTest,
6089 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006091 DataRate::BitsPerSec(kTargetBitrateBps),
6092 DataRate::BitsPerSec(kTargetBitrateBps),
6093 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006094
6095 const int kFrameWidth = 1280;
6096 const int kFrameHeight = 720;
6097 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006098 // requested by
6099 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006100 video_source_.set_adaptation_enabled(true);
6101
6102 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006103 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006104 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006105
6106 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006107 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006108 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006109 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006110 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006111
asaperssonfab67072017-04-04 05:51:49 -07006112 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006113 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006114 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006115 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006116 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006117
mflodmancc3d4422017-08-03 08:27:51 -07006118 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006119}
sprangfe627f32017-03-29 08:24:59 -07006120
mflodmancc3d4422017-08-03 08:27:51 -07006121TEST_F(VideoStreamEncoderTest,
6122 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006123 const int kFrameWidth = 1280;
6124 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006125
Henrik Boström381d1092020-05-12 18:49:07 +02006126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006127 DataRate::BitsPerSec(kTargetBitrateBps),
6128 DataRate::BitsPerSec(kTargetBitrateBps),
6129 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006130 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006131 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006132 video_source_.set_adaptation_enabled(true);
6133
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006134 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006135
6136 video_source_.IncomingCapturedFrame(
6137 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006138 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006139
6140 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006141 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006142
6143 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006144 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006145 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006146 video_source_.IncomingCapturedFrame(
6147 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006148 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006149 }
6150
6151 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006152 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006153 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006154 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006155 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006156 video_source_.IncomingCapturedFrame(
6157 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006158 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006159 ++num_frames_dropped;
6160 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006161 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006162 }
6163 }
6164
sprang4847ae62017-06-27 07:06:52 -07006165 // Add some slack to account for frames dropped by the frame dropper.
6166 const int kErrorMargin = 1;
6167 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006168 kErrorMargin);
6169
6170 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006171 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006172 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006173 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006174 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006175 video_source_.IncomingCapturedFrame(
6176 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006177 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006178 ++num_frames_dropped;
6179 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006180 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006181 }
6182 }
sprang4847ae62017-06-27 07:06:52 -07006183 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006184 kErrorMargin);
6185
6186 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006187 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006188 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006189 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006190 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006191 video_source_.IncomingCapturedFrame(
6192 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006193 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006194 ++num_frames_dropped;
6195 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006196 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006197 }
6198 }
sprang4847ae62017-06-27 07:06:52 -07006199 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006200 kErrorMargin);
6201
6202 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006203 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006204 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006205 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006206 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006207 video_source_.IncomingCapturedFrame(
6208 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006209 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006210 ++num_frames_dropped;
6211 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006212 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006213 }
6214 }
6215 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6216
mflodmancc3d4422017-08-03 08:27:51 -07006217 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006218}
6219
mflodmancc3d4422017-08-03 08:27:51 -07006220TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006221 const int kFramerateFps = 5;
6222 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006223 const int kFrameWidth = 1280;
6224 const int kFrameHeight = 720;
6225
sprang4847ae62017-06-27 07:06:52 -07006226 // Reconfigure encoder with two temporal layers and screensharing, which will
6227 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006228 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006229
Henrik Boström381d1092020-05-12 18:49:07 +02006230 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006231 DataRate::BitsPerSec(kTargetBitrateBps),
6232 DataRate::BitsPerSec(kTargetBitrateBps),
6233 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006234 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006235 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006236 video_source_.set_adaptation_enabled(true);
6237
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006238 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006239
6240 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006241 rtc::VideoSinkWants last_wants;
6242 do {
6243 last_wants = video_source_.sink_wants();
6244
sprangc5d62e22017-04-02 23:53:04 -07006245 // Insert frames to get a new fps estimate...
6246 for (int j = 0; j < kFramerateFps; ++j) {
6247 video_source_.IncomingCapturedFrame(
6248 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006249 if (video_source_.last_sent_width()) {
6250 sink_.WaitForEncodedFrame(timestamp_ms);
6251 }
sprangc5d62e22017-04-02 23:53:04 -07006252 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006253 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006254 }
6255 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006256 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006257 } while (video_source_.sink_wants().max_framerate_fps <
6258 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006259
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006260 EXPECT_THAT(video_source_.sink_wants(),
6261 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006262
mflodmancc3d4422017-08-03 08:27:51 -07006263 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006264}
asaperssonf7e294d2017-06-13 23:25:22 -07006265
mflodmancc3d4422017-08-03 08:27:51 -07006266TEST_F(VideoStreamEncoderTest,
6267 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006268 const int kWidth = 1280;
6269 const int kHeight = 720;
6270 const int64_t kFrameIntervalMs = 150;
6271 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006273 DataRate::BitsPerSec(kTargetBitrateBps),
6274 DataRate::BitsPerSec(kTargetBitrateBps),
6275 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006276
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006277 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006278 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006279 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006280 video_stream_encoder_->SetSource(&source,
6281 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006282 timestamp_ms += kFrameIntervalMs;
6283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006285 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006286 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6288 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6289
6290 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006292 timestamp_ms += kFrameIntervalMs;
6293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006294 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006295 EXPECT_THAT(source.sink_wants(),
6296 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6299 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6300
6301 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006302 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006303 timestamp_ms += kFrameIntervalMs;
6304 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006305 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006306 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6309 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6310
6311 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006312 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006313 timestamp_ms += kFrameIntervalMs;
6314 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006315 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006316 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6318 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6319 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6320
6321 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006322 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006323 timestamp_ms += kFrameIntervalMs;
6324 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006325 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006326 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6329 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6330
6331 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006332 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006333 timestamp_ms += kFrameIntervalMs;
6334 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006335 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006336 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6339 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6340
6341 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006342 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006343 timestamp_ms += kFrameIntervalMs;
6344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006345 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006346 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6349 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6350
6351 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006352 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006353 timestamp_ms += kFrameIntervalMs;
6354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006356 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006357 rtc::VideoSinkWants last_wants = source.sink_wants();
6358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6359 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6360 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6361
6362 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006363 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006364 timestamp_ms += kFrameIntervalMs;
6365 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006366 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006367 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006368 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6369 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6370 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6371
Evan Shrubsole64469032020-06-11 10:45:29 +02006372 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006373 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006374 timestamp_ms += kFrameIntervalMs;
6375 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006376 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006377 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006378 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6380 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6381
6382 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006383 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006384 timestamp_ms += kFrameIntervalMs;
6385 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006386 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006387 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006388 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6390 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6391
6392 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006393 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006394 timestamp_ms += kFrameIntervalMs;
6395 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006396 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006397 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006398 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6400 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6401
6402 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006403 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006404 timestamp_ms += kFrameIntervalMs;
6405 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006406 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006407 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006408 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6410 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6411
6412 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006413 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006414 timestamp_ms += kFrameIntervalMs;
6415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006416 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006417 EXPECT_THAT(source.sink_wants(), FpsMax());
6418 EXPECT_EQ(source.sink_wants().max_pixel_count,
6419 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006420 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6422 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6423
6424 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006425 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006426 timestamp_ms += kFrameIntervalMs;
6427 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006428 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006429 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6431 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6432 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6433
Åsa Persson30ab0152019-08-27 12:22:33 +02006434 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006435 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006436 timestamp_ms += kFrameIntervalMs;
6437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006438 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006439 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006440 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6442 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6443 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6444
6445 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006446 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006447 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006448 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6449
mflodmancc3d4422017-08-03 08:27:51 -07006450 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006451}
6452
mflodmancc3d4422017-08-03 08:27:51 -07006453TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006454 const int kWidth = 1280;
6455 const int kHeight = 720;
6456 const int64_t kFrameIntervalMs = 150;
6457 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006459 DataRate::BitsPerSec(kTargetBitrateBps),
6460 DataRate::BitsPerSec(kTargetBitrateBps),
6461 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006462
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006463 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006464 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006465 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006466 video_stream_encoder_->SetSource(&source,
6467 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006468 timestamp_ms += kFrameIntervalMs;
6469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006470 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006471 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006472 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6473 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6474 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6475 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6477 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6478
6479 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006480 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006481 timestamp_ms += kFrameIntervalMs;
6482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006483 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006484 EXPECT_THAT(source.sink_wants(),
6485 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6489 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6490 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6491 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6492
6493 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006494 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006495 timestamp_ms += kFrameIntervalMs;
6496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006497 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6501 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6502 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6503 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6504 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6505
6506 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006507 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006508 timestamp_ms += kFrameIntervalMs;
6509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006510 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006511 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6514 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6515 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6516 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6517 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6518
Evan Shrubsole64469032020-06-11 10:45:29 +02006519 // Trigger cpu adapt up, expect no change since QP is most limited.
6520 {
6521 // Store current sink wants since we expect no change and if there is no
6522 // change then last_wants() is not updated.
6523 auto previous_sink_wants = source.sink_wants();
6524 video_stream_encoder_->TriggerCpuUnderuse();
6525 timestamp_ms += kFrameIntervalMs;
6526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6527 WaitForEncodedFrame(timestamp_ms);
6528 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6529 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6530 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6531 }
6532
6533 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6534 video_stream_encoder_->TriggerQualityHigh();
6535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6537 WaitForEncodedFrame(timestamp_ms);
6538 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6541 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6542 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6543 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6544 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6545
6546 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6547 // expect increased resolution (960x540@30fps).
6548 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006549 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006550 timestamp_ms += kFrameIntervalMs;
6551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6556 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6557 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006559 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006560
Evan Shrubsole64469032020-06-11 10:45:29 +02006561 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6562 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006563 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006564 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006565 timestamp_ms += kFrameIntervalMs;
6566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006567 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006568 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006569 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6571 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6572 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6573 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6574 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006575 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006576
6577 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006578 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006579 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006580 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006581 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006582
mflodmancc3d4422017-08-03 08:27:51 -07006583 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006584}
6585
mflodmancc3d4422017-08-03 08:27:51 -07006586TEST_F(VideoStreamEncoderTest,
6587 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006588 const int kWidth = 640;
6589 const int kHeight = 360;
6590 const int kFpsLimit = 15;
6591 const int64_t kFrameIntervalMs = 150;
6592 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006594 DataRate::BitsPerSec(kTargetBitrateBps),
6595 DataRate::BitsPerSec(kTargetBitrateBps),
6596 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006597
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006598 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006599 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006600 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006601 video_stream_encoder_->SetSource(&source,
6602 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006603 timestamp_ms += kFrameIntervalMs;
6604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006605 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006606 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006607 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6608 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6609 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6610 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6611 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6612 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6613
6614 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006615 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006616 timestamp_ms += kFrameIntervalMs;
6617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006618 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006619 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6623 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6624 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6625 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6626
6627 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006628 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006631 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006632 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006633 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006635 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6636 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6637 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6638 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
Evan Shrubsole64469032020-06-11 10:45:29 +02006640 // Trigger cpu adapt up, expect no change because quality is most limited.
6641 {
6642 auto previous_sink_wants = source.sink_wants();
6643 // Store current sink wants since we expect no change ind if there is no
6644 // change then last__wants() is not updated.
6645 video_stream_encoder_->TriggerCpuUnderuse();
6646 timestamp_ms += kFrameIntervalMs;
6647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6648 WaitForEncodedFrame(timestamp_ms);
6649 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6650 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6651 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6652 }
6653
6654 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6655 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006656 timestamp_ms += kFrameIntervalMs;
6657 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006658 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006659 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006663 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6664 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6665 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006666
Evan Shrubsole64469032020-06-11 10:45:29 +02006667 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006668 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006669 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006670 timestamp_ms += kFrameIntervalMs;
6671 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006672 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006673 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6678 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006679 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006680
6681 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006682 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006684 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006685 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006686
mflodmancc3d4422017-08-03 08:27:51 -07006687 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006688}
6689
mflodmancc3d4422017-08-03 08:27:51 -07006690TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006691 const int kFrameWidth = 1920;
6692 const int kFrameHeight = 1080;
6693 // 3/4 of 1920.
6694 const int kAdaptedFrameWidth = 1440;
6695 // 3/4 of 1080 rounded down to multiple of 4.
6696 const int kAdaptedFrameHeight = 808;
6697 const int kFramerate = 24;
6698
Henrik Boström381d1092020-05-12 18:49:07 +02006699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006700 DataRate::BitsPerSec(kTargetBitrateBps),
6701 DataRate::BitsPerSec(kTargetBitrateBps),
6702 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006703 // Trigger reconfigure encoder (without resetting the entire instance).
6704 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006705 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6706 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006707 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006708 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006709 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006710 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006711 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006712 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006713
6714 video_source_.set_adaptation_enabled(true);
6715
6716 video_source_.IncomingCapturedFrame(
6717 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006718 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006719
6720 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006721 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006722 video_source_.IncomingCapturedFrame(
6723 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006724 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006725
mflodmancc3d4422017-08-03 08:27:51 -07006726 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006727}
6728
mflodmancc3d4422017-08-03 08:27:51 -07006729TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006730 const int kFrameWidth = 1280;
6731 const int kFrameHeight = 720;
6732 const int kLowFps = 2;
6733 const int kHighFps = 30;
6734
Henrik Boström381d1092020-05-12 18:49:07 +02006735 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006736 DataRate::BitsPerSec(kTargetBitrateBps),
6737 DataRate::BitsPerSec(kTargetBitrateBps),
6738 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006739
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006740 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006741 max_framerate_ = kLowFps;
6742
6743 // Insert 2 seconds of 2fps video.
6744 for (int i = 0; i < kLowFps * 2; ++i) {
6745 video_source_.IncomingCapturedFrame(
6746 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6747 WaitForEncodedFrame(timestamp_ms);
6748 timestamp_ms += 1000 / kLowFps;
6749 }
6750
6751 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006753 DataRate::BitsPerSec(kTargetBitrateBps),
6754 DataRate::BitsPerSec(kTargetBitrateBps),
6755 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006756 video_source_.IncomingCapturedFrame(
6757 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6758 WaitForEncodedFrame(timestamp_ms);
6759 timestamp_ms += 1000 / kLowFps;
6760
6761 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6762
6763 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006764 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006765 const int kFrameIntervalMs = 1000 / kHighFps;
6766 max_framerate_ = kHighFps;
6767 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6768 video_source_.IncomingCapturedFrame(
6769 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6770 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6771 // be dropped if the encoder hans't been updated with the new higher target
6772 // framerate yet, causing it to overshoot the target bitrate and then
6773 // suffering the wrath of the media optimizer.
6774 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6775 timestamp_ms += kFrameIntervalMs;
6776 }
6777
6778 // Don expect correct measurement just yet, but it should be higher than
6779 // before.
6780 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6781
mflodmancc3d4422017-08-03 08:27:51 -07006782 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006783}
6784
mflodmancc3d4422017-08-03 08:27:51 -07006785TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006786 const int kFrameWidth = 1280;
6787 const int kFrameHeight = 720;
6788 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006789 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006790 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006791 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006792
Henrik Boström381d1092020-05-12 18:49:07 +02006793 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006794 DataRate::BitsPerSec(kTargetBitrateBps),
6795 DataRate::BitsPerSec(kTargetBitrateBps),
6796 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006797 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006798
6799 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006800 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006801 video_source_.IncomingCapturedFrame(
6802 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6803 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006804 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006805
6806 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6808 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6809 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006810
6811 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006812 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006813 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006814
Per Kjellanderdcef6412020-10-07 15:09:05 +02006815 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006816 video_source_.IncomingCapturedFrame(
6817 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6818 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006819 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006820
mflodmancc3d4422017-08-03 08:27:51 -07006821 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006822}
ilnik6b826ef2017-06-16 06:53:48 -07006823
Niels Möller4db138e2018-04-19 09:04:13 +02006824TEST_F(VideoStreamEncoderTest,
6825 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6826 const int kFrameWidth = 1280;
6827 const int kFrameHeight = 720;
6828 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006830 DataRate::BitsPerSec(kTargetBitrateBps),
6831 DataRate::BitsPerSec(kTargetBitrateBps),
6832 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006833 video_source_.IncomingCapturedFrame(
6834 CreateFrame(1, kFrameWidth, kFrameHeight));
6835 WaitForEncodedFrame(1);
6836 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6837 .low_encode_usage_threshold_percent,
6838 default_options.low_encode_usage_threshold_percent);
6839 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6840 .high_encode_usage_threshold_percent,
6841 default_options.high_encode_usage_threshold_percent);
6842 video_stream_encoder_->Stop();
6843}
6844
6845TEST_F(VideoStreamEncoderTest,
6846 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6847 const int kFrameWidth = 1280;
6848 const int kFrameHeight = 720;
6849 CpuOveruseOptions hardware_options;
6850 hardware_options.low_encode_usage_threshold_percent = 150;
6851 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006852 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006853
Henrik Boström381d1092020-05-12 18:49:07 +02006854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006855 DataRate::BitsPerSec(kTargetBitrateBps),
6856 DataRate::BitsPerSec(kTargetBitrateBps),
6857 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006858 video_source_.IncomingCapturedFrame(
6859 CreateFrame(1, kFrameWidth, kFrameHeight));
6860 WaitForEncodedFrame(1);
6861 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6862 .low_encode_usage_threshold_percent,
6863 hardware_options.low_encode_usage_threshold_percent);
6864 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6865 .high_encode_usage_threshold_percent,
6866 hardware_options.high_encode_usage_threshold_percent);
6867 video_stream_encoder_->Stop();
6868}
6869
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006870TEST_F(VideoStreamEncoderTest,
6871 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6872 const int kFrameWidth = 1280;
6873 const int kFrameHeight = 720;
6874
6875 const CpuOveruseOptions default_options;
6876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6877 DataRate::BitsPerSec(kTargetBitrateBps),
6878 DataRate::BitsPerSec(kTargetBitrateBps),
6879 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6880 video_source_.IncomingCapturedFrame(
6881 CreateFrame(1, kFrameWidth, kFrameHeight));
6882 WaitForEncodedFrame(1);
6883 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6884 .low_encode_usage_threshold_percent,
6885 default_options.low_encode_usage_threshold_percent);
6886 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6887 .high_encode_usage_threshold_percent,
6888 default_options.high_encode_usage_threshold_percent);
6889
6890 CpuOveruseOptions hardware_options;
6891 hardware_options.low_encode_usage_threshold_percent = 150;
6892 hardware_options.high_encode_usage_threshold_percent = 200;
6893 fake_encoder_.SetIsHardwareAccelerated(true);
6894
6895 video_source_.IncomingCapturedFrame(
6896 CreateFrame(2, kFrameWidth, kFrameHeight));
6897 WaitForEncodedFrame(2);
6898
6899 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6900 .low_encode_usage_threshold_percent,
6901 hardware_options.low_encode_usage_threshold_percent);
6902 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6903 .high_encode_usage_threshold_percent,
6904 hardware_options.high_encode_usage_threshold_percent);
6905
6906 video_stream_encoder_->Stop();
6907}
6908
Niels Möller6bb5ab92019-01-11 11:11:10 +01006909TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6910 const int kFrameWidth = 320;
6911 const int kFrameHeight = 240;
6912 const int kFps = 30;
6913 const int kTargetBitrateBps = 120000;
6914 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6915
Henrik Boström381d1092020-05-12 18:49:07 +02006916 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006917 DataRate::BitsPerSec(kTargetBitrateBps),
6918 DataRate::BitsPerSec(kTargetBitrateBps),
6919 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006920
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006921 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006922 max_framerate_ = kFps;
6923
6924 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6925 fake_encoder_.SimulateOvershoot(1.0);
6926 int num_dropped = 0;
6927 for (int i = 0; i < kNumFramesInRun; ++i) {
6928 video_source_.IncomingCapturedFrame(
6929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6930 // Wait up to two frame durations for a frame to arrive.
6931 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6932 ++num_dropped;
6933 }
6934 timestamp_ms += 1000 / kFps;
6935 }
6936
Erik Språnga8d48ab2019-02-08 14:17:40 +01006937 // Framerate should be measured to be near the expected target rate.
6938 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6939
6940 // Frame drops should be within 5% of expected 0%.
6941 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006942
6943 // Make encoder produce frames at double the expected bitrate during 3 seconds
6944 // of video, verify number of drops. Rate needs to be slightly changed in
6945 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006946 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006947 const RateControlSettings trials =
6948 RateControlSettings::ParseFromFieldTrials();
6949 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006950 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006951 // frame dropping since the adjuter will try to just lower the target
6952 // bitrate rather than drop frames. If network headroom can be used, it
6953 // doesn't push back as hard so we don't need quite as much overshoot.
6954 // These numbers are unfortunately a bit magical but there's not trivial
6955 // way to algebraically infer them.
6956 if (trials.BitrateAdjusterCanUseNetworkHeadroom()) {
6957 overshoot_factor = 2.4;
6958 } else {
6959 overshoot_factor = 4.0;
6960 }
Erik Språng7ca375c2019-02-06 16:20:17 +01006961 }
6962 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006963 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006964 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6965 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6966 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006967 num_dropped = 0;
6968 for (int i = 0; i < kNumFramesInRun; ++i) {
6969 video_source_.IncomingCapturedFrame(
6970 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6971 // Wait up to two frame durations for a frame to arrive.
6972 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6973 ++num_dropped;
6974 }
6975 timestamp_ms += 1000 / kFps;
6976 }
6977
Henrik Boström381d1092020-05-12 18:49:07 +02006978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006979 DataRate::BitsPerSec(kTargetBitrateBps),
6980 DataRate::BitsPerSec(kTargetBitrateBps),
6981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006982
6983 // Target framerate should be still be near the expected target, despite
6984 // the frame drops.
6985 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6986
6987 // Frame drops should be within 5% of expected 50%.
6988 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006989
6990 video_stream_encoder_->Stop();
6991}
6992
6993TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6994 const int kFrameWidth = 320;
6995 const int kFrameHeight = 240;
6996 const int kActualInputFps = 24;
6997 const int kTargetBitrateBps = 120000;
6998
6999 ASSERT_GT(max_framerate_, kActualInputFps);
7000
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007001 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007002 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007004 DataRate::BitsPerSec(kTargetBitrateBps),
7005 DataRate::BitsPerSec(kTargetBitrateBps),
7006 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007007
7008 // Insert 3 seconds of video, with an input fps lower than configured max.
7009 for (int i = 0; i < kActualInputFps * 3; ++i) {
7010 video_source_.IncomingCapturedFrame(
7011 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7012 // Wait up to two frame durations for a frame to arrive.
7013 WaitForEncodedFrame(timestamp_ms);
7014 timestamp_ms += 1000 / kActualInputFps;
7015 }
7016
7017 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7018
7019 video_stream_encoder_->Stop();
7020}
7021
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007022TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007023 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007025 DataRate::BitsPerSec(kTargetBitrateBps),
7026 DataRate::BitsPerSec(kTargetBitrateBps),
7027 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007028
7029 fake_encoder_.BlockNextEncode();
7030 video_source_.IncomingCapturedFrame(
7031 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7032 WaitForEncodedFrame(1);
7033 // On the very first frame full update should be forced.
7034 rect = fake_encoder_.GetLastUpdateRect();
7035 EXPECT_EQ(rect.offset_x, 0);
7036 EXPECT_EQ(rect.offset_y, 0);
7037 EXPECT_EQ(rect.height, codec_height_);
7038 EXPECT_EQ(rect.width, codec_width_);
7039 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7040 // call to ContinueEncode.
7041 video_source_.IncomingCapturedFrame(
7042 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7043 ExpectDroppedFrame();
7044 video_source_.IncomingCapturedFrame(
7045 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7046 ExpectDroppedFrame();
7047 fake_encoder_.ContinueEncode();
7048 WaitForEncodedFrame(3);
7049 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7050 rect = fake_encoder_.GetLastUpdateRect();
7051 EXPECT_EQ(rect.offset_x, 1);
7052 EXPECT_EQ(rect.offset_y, 0);
7053 EXPECT_EQ(rect.width, 10);
7054 EXPECT_EQ(rect.height, 1);
7055
7056 video_source_.IncomingCapturedFrame(
7057 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7058 WaitForEncodedFrame(4);
7059 // Previous frame was encoded, so no accumulation should happen.
7060 rect = fake_encoder_.GetLastUpdateRect();
7061 EXPECT_EQ(rect.offset_x, 0);
7062 EXPECT_EQ(rect.offset_y, 0);
7063 EXPECT_EQ(rect.width, 1);
7064 EXPECT_EQ(rect.height, 1);
7065
7066 video_stream_encoder_->Stop();
7067}
7068
Erik Språngd7329ca2019-02-21 21:19:53 +01007069TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007071 DataRate::BitsPerSec(kTargetBitrateBps),
7072 DataRate::BitsPerSec(kTargetBitrateBps),
7073 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007074
7075 // First frame is always keyframe.
7076 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7077 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007078 EXPECT_THAT(
7079 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007080 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007081
7082 // Insert delta frame.
7083 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7084 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007085 EXPECT_THAT(
7086 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007087 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007088
7089 // Request next frame be a key-frame.
7090 video_stream_encoder_->SendKeyFrame();
7091 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7092 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007093 EXPECT_THAT(
7094 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007095 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007096
7097 video_stream_encoder_->Stop();
7098}
7099
7100TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7101 // Setup simulcast with three streams.
7102 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007104 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7105 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7106 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007107 // Wait for all three layers before triggering event.
7108 sink_.SetNumExpectedLayers(3);
7109
7110 // First frame is always keyframe.
7111 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7112 WaitForEncodedFrame(1);
7113 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007114 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7115 VideoFrameType::kVideoFrameKey,
7116 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007117
7118 // Insert delta frame.
7119 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7120 WaitForEncodedFrame(2);
7121 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007122 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7123 VideoFrameType::kVideoFrameDelta,
7124 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007125
7126 // Request next frame be a key-frame.
7127 // Only first stream is configured to produce key-frame.
7128 video_stream_encoder_->SendKeyFrame();
7129 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7130 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007131
7132 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7133 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007134 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007135 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007136 VideoFrameType::kVideoFrameKey,
7137 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007138
7139 video_stream_encoder_->Stop();
7140}
7141
7142TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7143 // Configure internal source factory and setup test again.
7144 encoder_factory_.SetHasInternalSource(true);
7145 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007147 DataRate::BitsPerSec(kTargetBitrateBps),
7148 DataRate::BitsPerSec(kTargetBitrateBps),
7149 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007150
7151 // Call encoder directly, simulating internal source where encoded frame
7152 // callback in VideoStreamEncoder is called despite no OnFrame().
7153 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7154 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007155 EXPECT_THAT(
7156 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007157 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007158
Niels Möller8f7ce222019-03-21 15:43:58 +01007159 const std::vector<VideoFrameType> kDeltaFrame = {
7160 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007161 // Need to set timestamp manually since manually for injected frame.
7162 VideoFrame frame = CreateFrame(101, nullptr);
7163 frame.set_timestamp(101);
7164 fake_encoder_.InjectFrame(frame, false);
7165 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007166 EXPECT_THAT(
7167 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007168 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007169
7170 // Request key-frame. The forces a dummy frame down into the encoder.
7171 fake_encoder_.ExpectNullFrame();
7172 video_stream_encoder_->SendKeyFrame();
7173 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007174 EXPECT_THAT(
7175 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007176 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007177
7178 video_stream_encoder_->Stop();
7179}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007180
7181TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7182 // Configure internal source factory and setup test again.
7183 encoder_factory_.SetHasInternalSource(true);
7184 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007185 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007186 DataRate::BitsPerSec(kTargetBitrateBps),
7187 DataRate::BitsPerSec(kTargetBitrateBps),
7188 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007189
7190 int64_t timestamp = 1;
7191 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007192 image.capture_time_ms_ = ++timestamp;
7193 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7194 const int64_t kEncodeFinishDelayMs = 10;
7195 image.timing_.encode_start_ms = timestamp;
7196 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007197 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007198 // Wait for frame without incrementing clock.
7199 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7200 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7201 // capture timestamp should be kEncodeFinishDelayMs in the past.
7202 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007203 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007204
7205 video_stream_encoder_->Stop();
7206}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007207
7208TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007209 // SPS contains VUI with restrictions on the maximum number of reordered
7210 // pictures, there is no need to rewrite the bitstream to enable faster
7211 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007212 ResetEncoder("H264", 1, 1, 1, false);
7213
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7215 DataRate::BitsPerSec(kTargetBitrateBps),
7216 DataRate::BitsPerSec(kTargetBitrateBps),
7217 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7218 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007219
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007220 fake_encoder_.SetEncodedImageData(
7221 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007222
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007223 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7224 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007225
7226 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7227 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007228
7229 video_stream_encoder_->Stop();
7230}
7231
7232TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007233 // SPS does not contain VUI, the bitstream is will be rewritten with added
7234 // VUI with restrictions on the maximum number of reordered pictures to
7235 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007236 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7237 0x00, 0x00, 0x03, 0x03, 0xF4,
7238 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007239 ResetEncoder("H264", 1, 1, 1, false);
7240
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007241 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7242 DataRate::BitsPerSec(kTargetBitrateBps),
7243 DataRate::BitsPerSec(kTargetBitrateBps),
7244 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7245 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007246
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007247 fake_encoder_.SetEncodedImageData(
7248 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007249
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007250 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7251 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007252
7253 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7254 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007255
7256 video_stream_encoder_->Stop();
7257}
7258
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007259TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7260 const int kFrameWidth = 1280;
7261 const int kFrameHeight = 720;
7262 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7263
Henrik Boström381d1092020-05-12 18:49:07 +02007264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007265 DataRate::BitsPerSec(kTargetBitrateBps),
7266 DataRate::BitsPerSec(kTargetBitrateBps),
7267 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007268 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7269
7270 // Insert a first video frame. It should be dropped because of downscale in
7271 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007272 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007273 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7274 frame.set_rotation(kVideoRotation_270);
7275 video_source_.IncomingCapturedFrame(frame);
7276
7277 ExpectDroppedFrame();
7278
7279 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007280 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007281 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7282 frame.set_rotation(kVideoRotation_90);
7283 video_source_.IncomingCapturedFrame(frame);
7284
7285 WaitForEncodedFrame(timestamp_ms);
7286 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7287
7288 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007289 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007290 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7291 frame.set_rotation(kVideoRotation_180);
7292 video_source_.IncomingCapturedFrame(frame);
7293
7294 WaitForEncodedFrame(timestamp_ms);
7295 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7296
7297 video_stream_encoder_->Stop();
7298}
7299
Erik Språng5056af02019-09-02 15:53:11 +02007300TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7301 const int kFrameWidth = 320;
7302 const int kFrameHeight = 180;
7303
7304 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007306 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7307 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7308 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007309 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007310 /*rtt_ms=*/0,
7311 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007312
7313 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007314 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007315 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7316 frame.set_rotation(kVideoRotation_270);
7317 video_source_.IncomingCapturedFrame(frame);
7318 WaitForEncodedFrame(timestamp_ms);
7319
7320 // Set a target rate below the minimum allowed by the codec settings.
7321 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007322 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7323 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007325 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007326 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007327 /*link_allocation=*/target_rate,
7328 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007329 /*rtt_ms=*/0,
7330 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7332
7333 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7334 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7335 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007336 DataRate allocation_sum =
7337 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007338 EXPECT_EQ(min_rate, allocation_sum);
7339 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7340
7341 video_stream_encoder_->Stop();
7342}
7343
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007344TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007346 DataRate::BitsPerSec(kTargetBitrateBps),
7347 DataRate::BitsPerSec(kTargetBitrateBps),
7348 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007349 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007350 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007351 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7352 WaitForEncodedFrame(1);
7353
7354 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7355 ASSERT_TRUE(prev_rate_settings.has_value());
7356 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7357 kDefaultFramerate);
7358
7359 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7360 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7361 timestamp_ms += 1000 / kDefaultFramerate;
7362 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7363 WaitForEncodedFrame(timestamp_ms);
7364 }
7365 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7366 kDefaultFramerate);
7367 // Capture larger frame to trigger a reconfigure.
7368 codec_height_ *= 2;
7369 codec_width_ *= 2;
7370 timestamp_ms += 1000 / kDefaultFramerate;
7371 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7372 WaitForEncodedFrame(timestamp_ms);
7373
7374 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7375 auto current_rate_settings =
7376 fake_encoder_.GetAndResetLastRateControlSettings();
7377 // Ensure we have actually reconfigured twice
7378 // The rate settings should have been set again even though
7379 // they haven't changed.
7380 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007381 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007382
7383 video_stream_encoder_->Stop();
7384}
7385
philipeld9cc8c02019-09-16 14:53:40 +02007386struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007387 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7388 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7389 MOCK_METHOD(void,
7390 RequestEncoderSwitch,
7391 (const webrtc::SdpVideoFormat& format),
7392 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007393};
7394
philipel9b058032020-02-10 11:30:00 +01007395TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7396 constexpr int kDontCare = 100;
7397 StrictMock<MockEncoderSelector> encoder_selector;
7398 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7399 &fake_encoder_, &encoder_selector);
7400 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7401
7402 // Reset encoder for new configuration to take effect.
7403 ConfigureEncoder(video_encoder_config_.Copy());
7404
7405 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7406
7407 video_source_.IncomingCapturedFrame(
7408 CreateFrame(kDontCare, kDontCare, kDontCare));
7409 video_stream_encoder_->Stop();
7410
7411 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7412 // to it's factory, so in order for the encoder instance in the
7413 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7414 // reset the |video_stream_encoder_| here.
7415 video_stream_encoder_.reset();
7416}
7417
7418TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7419 constexpr int kDontCare = 100;
7420
7421 NiceMock<MockEncoderSelector> encoder_selector;
7422 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7423 video_send_config_.encoder_settings.encoder_switch_request_callback =
7424 &switch_callback;
7425 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7426 &fake_encoder_, &encoder_selector);
7427 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7428
7429 // Reset encoder for new configuration to take effect.
7430 ConfigureEncoder(video_encoder_config_.Copy());
7431
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007432 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007433 .WillByDefault(Return(SdpVideoFormat("AV1")));
7434 EXPECT_CALL(switch_callback,
7435 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7436 Field(&SdpVideoFormat::name, "AV1"))));
7437
Henrik Boström381d1092020-05-12 18:49:07 +02007438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007439 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7440 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7441 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007442 /*fraction_lost=*/0,
7443 /*rtt_ms=*/0,
7444 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007445 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007446
7447 video_stream_encoder_->Stop();
7448}
7449
7450TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7451 constexpr int kSufficientBitrateToNotDrop = 1000;
7452 constexpr int kDontCare = 100;
7453
7454 NiceMock<MockVideoEncoder> video_encoder;
7455 NiceMock<MockEncoderSelector> encoder_selector;
7456 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7457 video_send_config_.encoder_settings.encoder_switch_request_callback =
7458 &switch_callback;
7459 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7460 &video_encoder, &encoder_selector);
7461 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7462
7463 // Reset encoder for new configuration to take effect.
7464 ConfigureEncoder(video_encoder_config_.Copy());
7465
7466 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7467 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7468 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007470 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7471 /*stable_target_bitrate=*/
7472 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7473 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007474 /*fraction_lost=*/0,
7475 /*rtt_ms=*/0,
7476 /*cwnd_reduce_ratio=*/0);
7477
7478 ON_CALL(video_encoder, Encode(_, _))
7479 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7480 ON_CALL(encoder_selector, OnEncoderBroken())
7481 .WillByDefault(Return(SdpVideoFormat("AV2")));
7482
7483 rtc::Event encode_attempted;
7484 EXPECT_CALL(switch_callback,
7485 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7486 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7487 EXPECT_EQ(format.name, "AV2");
7488 encode_attempted.Set();
7489 });
7490
7491 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7492 encode_attempted.Wait(3000);
7493
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007494 AdvanceTime(TimeDelta::Millis(0));
7495
philipel9b058032020-02-10 11:30:00 +01007496 video_stream_encoder_->Stop();
7497
7498 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7499 // to it's factory, so in order for the encoder instance in the
7500 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7501 // reset the |video_stream_encoder_| here.
7502 video_stream_encoder_.reset();
7503}
7504
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007505TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007506 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007507 const int kFrameWidth = 320;
7508 const int kFrameHeight = 180;
7509
7510 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007511 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007513 /*target_bitrate=*/rate,
7514 /*stable_target_bitrate=*/rate,
7515 /*link_allocation=*/rate,
7516 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007517 /*rtt_ms=*/0,
7518 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007519
7520 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007521 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007522 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7523 frame.set_rotation(kVideoRotation_270);
7524 video_source_.IncomingCapturedFrame(frame);
7525 WaitForEncodedFrame(timestamp_ms);
7526 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7527
7528 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007529 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007531 /*target_bitrate=*/new_stable_rate,
7532 /*stable_target_bitrate=*/new_stable_rate,
7533 /*link_allocation=*/rate,
7534 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007535 /*rtt_ms=*/0,
7536 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007537 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7538 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7539 video_stream_encoder_->Stop();
7540}
7541
7542TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007543 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007544 const int kFrameWidth = 320;
7545 const int kFrameHeight = 180;
7546
7547 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007548 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007549 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007550 /*target_bitrate=*/rate,
7551 /*stable_target_bitrate=*/rate,
7552 /*link_allocation=*/rate,
7553 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007554 /*rtt_ms=*/0,
7555 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007556
7557 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007558 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007559 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7560 frame.set_rotation(kVideoRotation_270);
7561 video_source_.IncomingCapturedFrame(frame);
7562 WaitForEncodedFrame(timestamp_ms);
7563 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7564
7565 // Set a higher target rate without changing the link_allocation. Should not
7566 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007567 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007569 /*target_bitrate=*/rate,
7570 /*stable_target_bitrate=*/new_stable_rate,
7571 /*link_allocation=*/rate,
7572 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007573 /*rtt_ms=*/0,
7574 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7576 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7577 video_stream_encoder_->Stop();
7578}
7579
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007580TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7581 test::ScopedFieldTrials field_trials(
7582 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7583 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7584 const int kFramerateFps = 30;
7585 const int kWidth = 1920;
7586 const int kHeight = 1080;
7587 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7588 // Works on screenshare mode.
7589 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7590 // We rely on the automatic resolution adaptation, but we handle framerate
7591 // adaptation manually by mocking the stats proxy.
7592 video_source_.set_adaptation_enabled(true);
7593
7594 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007596 DataRate::BitsPerSec(kTargetBitrateBps),
7597 DataRate::BitsPerSec(kTargetBitrateBps),
7598 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007599 video_stream_encoder_->SetSource(&video_source_,
7600 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007601 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007602
7603 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7604 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7605
7606 // Pass enough frames with the full update to trigger animation detection.
7607 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007608 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007609 frame.set_ntp_time_ms(timestamp_ms);
7610 frame.set_timestamp_us(timestamp_ms * 1000);
7611 video_source_.IncomingCapturedFrame(frame);
7612 WaitForEncodedFrame(timestamp_ms);
7613 }
7614
7615 // Resolution should be limited.
7616 rtc::VideoSinkWants expected;
7617 expected.max_framerate_fps = kFramerateFps;
7618 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007619 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007620
7621 // Pass one frame with no known update.
7622 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007623 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007624 frame.set_ntp_time_ms(timestamp_ms);
7625 frame.set_timestamp_us(timestamp_ms * 1000);
7626 frame.clear_update_rect();
7627
7628 video_source_.IncomingCapturedFrame(frame);
7629 WaitForEncodedFrame(timestamp_ms);
7630
7631 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007632 EXPECT_THAT(video_source_.sink_wants(),
7633 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007634
7635 video_stream_encoder_->Stop();
7636}
7637
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007638TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7639 const int kWidth = 720; // 540p adapted down.
7640 const int kHeight = 405;
7641 const int kNumFrames = 3;
7642 // Works on screenshare mode.
7643 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7644 /*num_spatial_layers=*/2, /*screenshare=*/true);
7645
7646 video_source_.set_adaptation_enabled(true);
7647
7648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7649 DataRate::BitsPerSec(kTargetBitrateBps),
7650 DataRate::BitsPerSec(kTargetBitrateBps),
7651 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7652
7653 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7654
7655 // Pass enough frames with the full update to trigger animation detection.
7656 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007657 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007658 frame.set_ntp_time_ms(timestamp_ms);
7659 frame.set_timestamp_us(timestamp_ms * 1000);
7660 video_source_.IncomingCapturedFrame(frame);
7661 WaitForEncodedFrame(timestamp_ms);
7662 }
7663
7664 video_stream_encoder_->Stop();
7665}
7666
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007667TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7668 const float downscale_factors[] = {4.0, 2.0, 1.0};
7669 const int number_layers =
7670 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7671 VideoEncoderConfig config;
7672 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7673 for (int i = 0; i < number_layers; ++i) {
7674 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7675 config.simulcast_layers[i].active = true;
7676 }
7677 config.video_stream_factory =
7678 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7679 "VP8", /*max qp*/ 56, /*screencast*/ false,
7680 /*screenshare enabled*/ false);
7681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7682 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7683 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7684 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7685
7686 // First initialization.
7687 // Encoder should be initialized. Next frame should be key frame.
7688 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7689 sink_.SetNumExpectedLayers(number_layers);
7690 int64_t timestamp_ms = kFrameIntervalMs;
7691 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7692 WaitForEncodedFrame(timestamp_ms);
7693 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7694 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7695 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7696 VideoFrameType::kVideoFrameKey,
7697 VideoFrameType::kVideoFrameKey}));
7698
7699 // Disable top layer.
7700 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7701 config.simulcast_layers[number_layers - 1].active = false;
7702 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7703 sink_.SetNumExpectedLayers(number_layers - 1);
7704 timestamp_ms += kFrameIntervalMs;
7705 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7706 WaitForEncodedFrame(timestamp_ms);
7707 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7708 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7709 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7710 VideoFrameType::kVideoFrameDelta,
7711 VideoFrameType::kVideoFrameDelta}));
7712
7713 // Re-enable top layer.
7714 // Encoder should be re-initialized. Next frame should be key frame.
7715 config.simulcast_layers[number_layers - 1].active = true;
7716 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7717 sink_.SetNumExpectedLayers(number_layers);
7718 timestamp_ms += kFrameIntervalMs;
7719 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7720 WaitForEncodedFrame(timestamp_ms);
7721 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7722 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7723 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7724 VideoFrameType::kVideoFrameKey,
7725 VideoFrameType::kVideoFrameKey}));
7726
7727 // Top layer max rate change.
7728 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7729 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7730 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7731 sink_.SetNumExpectedLayers(number_layers);
7732 timestamp_ms += kFrameIntervalMs;
7733 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7734 WaitForEncodedFrame(timestamp_ms);
7735 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7736 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7737 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7738 VideoFrameType::kVideoFrameDelta,
7739 VideoFrameType::kVideoFrameDelta}));
7740
7741 // Top layer resolution change.
7742 // Encoder should be re-initialized. Next frame should be key frame.
7743 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7744 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7745 sink_.SetNumExpectedLayers(number_layers);
7746 timestamp_ms += kFrameIntervalMs;
7747 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7748 WaitForEncodedFrame(timestamp_ms);
7749 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7750 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7751 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7752 VideoFrameType::kVideoFrameKey,
7753 VideoFrameType::kVideoFrameKey}));
7754 video_stream_encoder_->Stop();
7755}
7756
Henrik Boström1124ed12021-02-25 10:30:39 +01007757TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7758 const int kFrameWidth = 1280;
7759 const int kFrameHeight = 720;
7760
7761 SetUp();
7762 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7763 DataRate::BitsPerSec(kTargetBitrateBps),
7764 DataRate::BitsPerSec(kTargetBitrateBps),
7765 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7766
7767 // Capturing a frame should reconfigure the encoder and expose the encoder
7768 // resolution, which is the same as the input frame.
7769 int64_t timestamp_ms = kFrameIntervalMs;
7770 video_source_.IncomingCapturedFrame(
7771 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7772 WaitForEncodedFrame(timestamp_ms);
7773 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7774 EXPECT_THAT(video_source_.sink_wants().resolutions,
7775 ::testing::ElementsAreArray(
7776 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7777
7778 video_stream_encoder_->Stop();
7779}
7780
7781TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7782 // Pick downscale factors such that we never encode at full resolution - this
7783 // is an interesting use case. The frame resolution influences the encoder
7784 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7785 // encoder should not ask for the frame resolution. This allows video frames
7786 // to have the appearence of one resolution but optimize its internal buffers
7787 // for what is actually encoded.
7788 const size_t kNumSimulcastLayers = 3u;
7789 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7790 const int kFrameWidth = 1280;
7791 const int kFrameHeight = 720;
7792 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7793 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7794 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7795 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7796 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7797 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7798
7799 VideoEncoderConfig config;
7800 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7801 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7802 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7803 config.simulcast_layers[i].active = true;
7804 }
7805 config.video_stream_factory =
7806 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7807 "VP8", /*max qp*/ 56, /*screencast*/ false,
7808 /*screenshare enabled*/ false);
7809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7810 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7811 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7812 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7813
7814 // Capture a frame with all layers active.
7815 int64_t timestamp_ms = kFrameIntervalMs;
7816 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7817 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7818 video_source_.IncomingCapturedFrame(
7819 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7820 WaitForEncodedFrame(timestamp_ms);
7821 // Expect encoded resolutions to match the expected simulcast layers.
7822 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7823 EXPECT_THAT(
7824 video_source_.sink_wants().resolutions,
7825 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7826
7827 // Capture a frame with one of the layers inactive.
7828 timestamp_ms += kFrameIntervalMs;
7829 config.simulcast_layers[2].active = false;
7830 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7831 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7832 video_source_.IncomingCapturedFrame(
7833 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7834 WaitForEncodedFrame(timestamp_ms);
7835
7836 // Expect encoded resolutions to match the expected simulcast layers.
7837 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7838 EXPECT_THAT(video_source_.sink_wants().resolutions,
7839 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7840
7841 // Capture a frame with all but one layer turned off.
7842 timestamp_ms += kFrameIntervalMs;
7843 config.simulcast_layers[1].active = false;
7844 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7845 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7846 video_source_.IncomingCapturedFrame(
7847 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7848 WaitForEncodedFrame(timestamp_ms);
7849
7850 // Expect encoded resolutions to match the expected simulcast layers.
7851 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7852 EXPECT_THAT(video_source_.sink_wants().resolutions,
7853 ::testing::ElementsAreArray({kLayer0Size}));
7854
7855 video_stream_encoder_->Stop();
7856}
7857
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007858TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
7859 // Enable encoder source to force encoder reconfig.
7860 encoder_factory_.SetHasInternalSource(true);
7861 ResetEncoder("VP8", 1, 1, 1, false);
7862
7863 // Set QP on encoded frame and pass the frame to encode complete callback.
7864 // Since QP is present QP parsing won't be triggered and the original value
7865 // should be kept.
7866 EncodedImage encoded_image;
7867 encoded_image.qp_ = 123;
7868 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7869 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7870 CodecSpecificInfo codec_info;
7871 codec_info.codecType = kVideoCodecVP8;
7872 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7873 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7874 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7875 video_stream_encoder_->Stop();
7876}
7877
7878TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
7879 // Enable encoder source to force encoder reconfig.
7880 encoder_factory_.SetHasInternalSource(true);
7881 ResetEncoder("VP8", 1, 1, 1, false);
7882
7883 // Pass an encoded frame without QP to encode complete callback. QP should be
7884 // parsed and set.
7885 EncodedImage encoded_image;
7886 encoded_image.qp_ = -1;
7887 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7888 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7889 CodecSpecificInfo codec_info;
7890 codec_info.codecType = kVideoCodecVP8;
7891 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7892 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7893 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7894 video_stream_encoder_->Stop();
7895}
7896
7897TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7898 webrtc::test::ScopedFieldTrials field_trials(
7899 "WebRTC-QpParsingKillSwitch/Enabled/");
7900
7901 // Enable encoder source to force encoder reconfig.
7902 encoder_factory_.SetHasInternalSource(true);
7903 ResetEncoder("VP8", 1, 1, 1, false);
7904
7905 EncodedImage encoded_image;
7906 encoded_image.qp_ = -1;
7907 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7908 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7909 CodecSpecificInfo codec_info;
7910 codec_info.codecType = kVideoCodecVP8;
7911 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7912 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7913 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7914 video_stream_encoder_->Stop();
7915}
7916
Sergey Silkind19e3b92021-03-16 10:05:30 +00007917TEST_F(VideoStreamEncoderTest,
7918 QualityScalingNotAllowed_QualityScalingDisabled) {
7919 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7920
7921 // Disable scaling settings in encoder info.
7922 fake_encoder_.SetQualityScaling(false);
7923 // Disable quality scaling in encoder config.
7924 video_encoder_config.is_quality_scaling_allowed = false;
7925 ConfigureEncoder(std::move(video_encoder_config));
7926
7927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7928 DataRate::BitsPerSec(kTargetBitrateBps),
7929 DataRate::BitsPerSec(kTargetBitrateBps),
7930 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7931
7932 test::FrameForwarder source;
7933 video_stream_encoder_->SetSource(
7934 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7935 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7936 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7937
7938 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7939 WaitForEncodedFrame(1);
7940 video_stream_encoder_->TriggerQualityLow();
7941 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7942
7943 video_stream_encoder_->Stop();
7944}
7945
7946#if !defined(WEBRTC_IOS)
7947// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
7948// disabled by default on iOS.
7949TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
7950 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7951
7952 // Disable scaling settings in encoder info.
7953 fake_encoder_.SetQualityScaling(false);
7954 // Enable quality scaling in encoder config.
7955 video_encoder_config.is_quality_scaling_allowed = true;
7956 ConfigureEncoder(std::move(video_encoder_config));
7957
7958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7959 DataRate::BitsPerSec(kTargetBitrateBps),
7960 DataRate::BitsPerSec(kTargetBitrateBps),
7961 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7962
7963 test::FrameForwarder source;
7964 video_stream_encoder_->SetSource(
7965 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7966 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7967 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7968
7969 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7970 WaitForEncodedFrame(1);
7971 video_stream_encoder_->TriggerQualityLow();
7972 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
7973
7974 video_stream_encoder_->Stop();
7975}
7976#endif
7977
Henrik Boström56db9ff2021-03-24 09:06:45 +01007978// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
7979class VideoStreamEncoderWithRealEncoderTest
7980 : public VideoStreamEncoderTest,
7981 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
7982 public:
7983 VideoStreamEncoderWithRealEncoderTest()
7984 : VideoStreamEncoderTest(),
7985 codec_type_(std::get<0>(GetParam())),
7986 allow_i420_conversion_(std::get<1>(GetParam())) {}
7987
7988 void SetUp() override {
7989 VideoStreamEncoderTest::SetUp();
7990 std::unique_ptr<VideoEncoder> encoder;
7991 switch (codec_type_) {
7992 case kVideoCodecVP8:
7993 encoder = VP8Encoder::Create();
7994 break;
7995 case kVideoCodecVP9:
7996 encoder = VP9Encoder::Create();
7997 break;
7998 case kVideoCodecAV1:
7999 encoder = CreateLibaomAv1Encoder();
8000 break;
8001 case kVideoCodecH264:
8002 encoder =
8003 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8004 break;
8005 case kVideoCodecMultiplex:
8006 mock_encoder_factory_for_multiplex_ =
8007 std::make_unique<MockVideoEncoderFactory>();
8008 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8009 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8010 .WillRepeatedly([] { return VP8Encoder::Create(); });
8011 encoder = std::make_unique<MultiplexEncoderAdapter>(
8012 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8013 false);
8014 break;
8015 default:
8016 RTC_NOTREACHED();
8017 }
8018 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8019 }
8020
8021 void TearDown() override {
8022 video_stream_encoder_->Stop();
8023 // Ensure |video_stream_encoder_| is destroyed before
8024 // |encoder_proxy_factory_|.
8025 video_stream_encoder_.reset();
8026 VideoStreamEncoderTest::TearDown();
8027 }
8028
8029 protected:
8030 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8031 std::unique_ptr<VideoEncoder> encoder) {
8032 // Configure VSE to use the encoder.
8033 encoder_ = std::move(encoder);
8034 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8035 encoder_.get(), &encoder_selector_);
8036 video_send_config_.encoder_settings.encoder_factory =
8037 encoder_proxy_factory_.get();
8038 VideoEncoderConfig video_encoder_config;
8039 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8040 video_encoder_config_ = video_encoder_config.Copy();
8041 ConfigureEncoder(video_encoder_config_.Copy());
8042
8043 // Set bitrate to ensure frame is not dropped.
8044 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8045 DataRate::BitsPerSec(kTargetBitrateBps),
8046 DataRate::BitsPerSec(kTargetBitrateBps),
8047 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8048 }
8049
8050 const VideoCodecType codec_type_;
8051 const bool allow_i420_conversion_;
8052 NiceMock<MockEncoderSelector> encoder_selector_;
8053 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8054 std::unique_ptr<VideoEncoder> encoder_;
8055 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8056};
8057
8058TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8059 auto native_i420_frame = test::CreateMappableNativeFrame(
8060 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8061 video_source_.IncomingCapturedFrame(native_i420_frame);
8062 WaitForEncodedFrame(codec_width_, codec_height_);
8063
8064 auto mappable_native_buffer =
8065 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8066 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8067 mappable_native_buffer->GetMappedFramedBuffers();
8068 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8069 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8070 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8071 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8072}
8073
8074TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8075 auto native_nv12_frame = test::CreateMappableNativeFrame(
8076 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8077 video_source_.IncomingCapturedFrame(native_nv12_frame);
8078 WaitForEncodedFrame(codec_width_, codec_height_);
8079
8080 auto mappable_native_buffer =
8081 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8082 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8083 mappable_native_buffer->GetMappedFramedBuffers();
8084 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8085 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8086 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8087 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8088
8089 if (!allow_i420_conversion_) {
8090 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8091 }
8092}
8093
8094std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8095 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8096 VideoCodecType codec_type = std::get<0>(info.param);
8097 bool allow_i420_conversion = std::get<1>(info.param);
8098 std::string str;
8099 switch (codec_type) {
8100 case kVideoCodecGeneric:
8101 str = "Generic";
8102 break;
8103 case kVideoCodecVP8:
8104 str = "VP8";
8105 break;
8106 case kVideoCodecVP9:
8107 str = "VP9";
8108 break;
8109 case kVideoCodecAV1:
8110 str = "AV1";
8111 break;
8112 case kVideoCodecH264:
8113 str = "H264";
8114 break;
8115 case kVideoCodecMultiplex:
8116 str = "Multiplex";
8117 break;
8118 default:
8119 RTC_NOTREACHED();
8120 }
8121 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8122 return str;
8123}
8124
8125constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8126 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8127constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8128 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8129constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8130 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8131constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8132 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8133#if defined(WEBRTC_USE_H264)
8134constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8135 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8136
8137// The windows compiler does not tolerate #if statements inside the
8138// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8139// and without H264).
8140INSTANTIATE_TEST_SUITE_P(
8141 All,
8142 VideoStreamEncoderWithRealEncoderTest,
8143 ::testing::Values(kVP8DisallowConversion,
8144 kVP9DisallowConversion,
8145 kAV1AllowConversion,
8146 kMultiplexDisallowConversion,
8147 kH264AllowConversion),
8148 TestParametersVideoCodecAndAllowI420ConversionToString);
8149#else
8150INSTANTIATE_TEST_SUITE_P(
8151 All,
8152 VideoStreamEncoderWithRealEncoderTest,
8153 ::testing::Values(kVP8DisallowConversion,
8154 kVP9DisallowConversion,
8155 kAV1AllowConversion,
8156 kMultiplexDisallowConversion),
8157 TestParametersVideoCodecAndAllowI420ConversionToString);
8158#endif
8159
perkj26091b12016-09-01 01:17:40 -07008160} // namespace webrtc