blob: 7ff3d4dca1871fa1b358e62433bd56b053ae8583 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020021#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010022#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010023#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080024#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020026#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020027#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010028#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010029#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020030#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020031#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010032#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020033#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010034#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020035#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070036#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080037#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020038#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010039#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
40#include "modules/video_coding/codecs/h264/include/h264.h"
41#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
42#include "modules/video_coding/codecs/vp8/include/vp8.h"
43#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020044#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020045#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010046#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020047#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010048#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020049#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020050#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080051#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020052#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010053#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020054#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "test/encoder_settings.h"
56#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020057#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010058#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "test/gmock.h"
60#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010061#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020062#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020063#include "test/video_encoder_proxy_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020064#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070065
66namespace webrtc {
67
sprang57c2fff2017-01-16 06:24:02 -080068using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020069using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020070using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020071using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020073using ::testing::Ge;
74using ::testing::Gt;
75using ::testing::Le;
76using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010077using ::testing::Matcher;
78using ::testing::NiceMock;
79using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010080using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020081using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080082
perkj803d97f2016-11-01 11:45:46 -070083namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020084const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010085const int kQpLow = 1;
86const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020087const int kMinFramerateFps = 2;
88const int kMinBalancedFramerateFps = 7;
89const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080090const size_t kMaxPayloadLength = 1440;
Erik Språngd7329ca2019-02-21 21:19:53 +010091const uint32_t kTargetBitrateBps = 1000000;
Sergey Silkin5ee69672019-07-02 14:18:34 +020092const uint32_t kStartBitrateBps = 600000;
Erik Språngd7329ca2019-02-21 21:19:53 +010093const uint32_t kSimulcastTargetBitrateBps = 3150000;
94const uint32_t kLowTargetBitrateBps = kTargetBitrateBps / 10;
kthelgason2bc68642017-02-07 07:02:22 -080095const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -070096const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +020097const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +020098const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +020099const VideoEncoder::ResolutionBitrateLimits
100 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
101const VideoEncoder::ResolutionBitrateLimits
102 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800103
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200104uint8_t optimal_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
105 0x00, 0x00, 0x03, 0x03, 0xF4,
106 0x05, 0x03, 0xC7, 0xE0, 0x1B,
107 0x41, 0x10, 0x8D, 0x00};
108
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100109const uint8_t kCodedFrameVp8Qp25[] = {
110 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
111 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
112 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
113
perkj803d97f2016-11-01 11:45:46 -0700114class TestBuffer : public webrtc::I420Buffer {
115 public:
116 TestBuffer(rtc::Event* event, int width, int height)
117 : I420Buffer(width, height), event_(event) {}
118
119 private:
120 friend class rtc::RefCountedObject<TestBuffer>;
121 ~TestBuffer() override {
122 if (event_)
123 event_->Set();
124 }
125 rtc::Event* const event_;
126};
127
Henrik Boström56db9ff2021-03-24 09:06:45 +0100128// A fake native buffer that can't be converted to I420. Upon scaling, it
129// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700130class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
131 public:
132 FakeNativeBuffer(rtc::Event* event, int width, int height)
133 : event_(event), width_(width), height_(height) {}
134 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
135 int width() const override { return width_; }
136 int height() const override { return height_; }
137 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
138 return nullptr;
139 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100140 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
141 int offset_x,
142 int offset_y,
143 int crop_width,
144 int crop_height,
145 int scaled_width,
146 int scaled_height) override {
147 return new rtc::RefCountedObject<FakeNativeBuffer>(nullptr, scaled_width,
148 scaled_height);
149 }
Noah Richards51db4212019-06-12 06:59:12 -0700150
151 private:
152 friend class rtc::RefCountedObject<FakeNativeBuffer>;
153 ~FakeNativeBuffer() override {
154 if (event_)
155 event_->Set();
156 }
157 rtc::Event* const event_;
158 const int width_;
159 const int height_;
160};
161
Evan Shrubsole895556e2020-10-05 09:15:13 +0200162// A fake native buffer that is backed by an NV12 buffer.
163class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
164 public:
165 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
166 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
167
168 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
169 int width() const override { return nv12_buffer_->width(); }
170 int height() const override { return nv12_buffer_->height(); }
171 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
172 return nv12_buffer_->ToI420();
173 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200174 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
175 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
176 if (absl::c_find(types, Type::kNV12) != types.end()) {
177 return nv12_buffer_;
178 }
179 return nullptr;
180 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200181 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
182
183 private:
184 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
185 ~FakeNV12NativeBuffer() override {
186 if (event_)
187 event_->Set();
188 }
189 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
190 rtc::Event* const event_;
191};
192
Niels Möller7dc26b72017-12-06 10:27:48 +0100193class CpuOveruseDetectorProxy : public OveruseFrameDetector {
194 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200195 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
196 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200197 last_target_framerate_fps_(-1),
198 framerate_updated_event_(true /* manual_reset */,
199 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100200 virtual ~CpuOveruseDetectorProxy() {}
201
202 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200203 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100204 last_target_framerate_fps_ = framerate_fps;
205 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200206 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100207 }
208
209 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200210 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100211 return last_target_framerate_fps_;
212 }
213
Niels Möller4db138e2018-04-19 09:04:13 +0200214 CpuOveruseOptions GetOptions() { return options_; }
215
Henrik Boström381d1092020-05-12 18:49:07 +0200216 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
217
Niels Möller7dc26b72017-12-06 10:27:48 +0100218 private:
Markus Handella3765182020-07-08 13:13:32 +0200219 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100220 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200221 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100222};
223
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200224class FakeVideoSourceRestrictionsListener
225 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200226 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200227 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200228 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200229 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200230 RTC_DCHECK(was_restrictions_updated_);
231 }
232
233 rtc::Event* restrictions_updated_event() {
234 return &restrictions_updated_event_;
235 }
236
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200237 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200238 void OnVideoSourceRestrictionsUpdated(
239 VideoSourceRestrictions restrictions,
240 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200241 rtc::scoped_refptr<Resource> reason,
242 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200243 was_restrictions_updated_ = true;
244 restrictions_updated_event_.Set();
245 }
246
247 private:
248 bool was_restrictions_updated_;
249 rtc::Event restrictions_updated_event_;
250};
251
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200252auto WantsFps(Matcher<int> fps_matcher) {
253 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
254 fps_matcher);
255}
256
257auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
258 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
259 AllOf(max_pixel_matcher, Gt(0)));
260}
261
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200262auto ResolutionMax() {
263 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200264 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200265 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
266 Eq(absl::nullopt)));
267}
268
269auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200270 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200271}
272
273auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200274 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200275}
276
277auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200278 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200279}
280
281auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200282 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200283}
284
285auto FpsMaxResolutionMax() {
286 return AllOf(FpsMax(), ResolutionMax());
287}
288
289auto UnlimitedSinkWants() {
290 return AllOf(FpsUnlimited(), ResolutionMax());
291}
292
293auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
294 Matcher<int> fps_range_matcher;
295
296 if (last_frame_pixels <= 320 * 240) {
297 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200298 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200299 fps_range_matcher = AllOf(Ge(10), Le(15));
300 } else if (last_frame_pixels <= 640 * 480) {
301 fps_range_matcher = Ge(15);
302 } else {
303 fps_range_matcher = Eq(kDefaultFramerate);
304 }
305 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
306 fps_range_matcher);
307}
308
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
310 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
311 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
312}
313
314auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
315 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
316}
317
318auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
319 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
320}
321
322auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
323 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
324 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
325}
326
327auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
328 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
329 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
330}
331
332auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
333 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
334 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
335}
336
337auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
338 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
339 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
340}
341
mflodmancc3d4422017-08-03 08:27:51 -0700342class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700343 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200344 VideoStreamEncoderUnderTest(TimeController* time_controller,
345 TaskQueueFactory* task_queue_factory,
346 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100347 const VideoStreamEncoderSettings& settings,
348 VideoStreamEncoder::BitrateAllocationCallbackType
349 allocation_callback_type)
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200350 : VideoStreamEncoder(time_controller->GetClock(),
Sebastian Jansson572c60f2019-03-04 18:30:41 +0100351 1 /* number_of_cores */,
Yves Gerey665174f2018-06-19 15:03:05 +0200352 stats_proxy,
353 settings,
Yves Gerey665174f2018-06-19 15:03:05 +0200354 std::unique_ptr<OveruseFrameDetector>(
355 overuse_detector_proxy_ =
Sebastian Jansson74682c12019-03-01 11:50:20 +0100356 new CpuOveruseDetectorProxy(stats_proxy)),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100357 task_queue_factory,
358 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200359 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200360 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200361 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200362 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200363 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200364 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200365 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200366 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100367 }
perkj803d97f2016-11-01 11:45:46 -0700368
Henrik Boström381d1092020-05-12 18:49:07 +0200369 void SetSourceAndWaitForRestrictionsUpdated(
370 rtc::VideoSourceInterface<VideoFrame>* source,
371 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200372 FakeVideoSourceRestrictionsListener listener;
373 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200374 SetSource(source, degradation_preference);
375 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200376 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200377 }
378
379 void SetSourceAndWaitForFramerateUpdated(
380 rtc::VideoSourceInterface<VideoFrame>* source,
381 const DegradationPreference& degradation_preference) {
382 overuse_detector_proxy_->framerate_updated_event()->Reset();
383 SetSource(source, degradation_preference);
384 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
385 }
386
387 void OnBitrateUpdatedAndWaitForManagedResources(
388 DataRate target_bitrate,
389 DataRate stable_target_bitrate,
390 DataRate link_allocation,
391 uint8_t fraction_lost,
392 int64_t round_trip_time_ms,
393 double cwnd_reduce_ratio) {
394 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
395 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
396 // Bitrate is updated on the encoder queue.
397 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200398 }
399
kthelgason2fc52542017-03-03 00:24:41 -0800400 // This is used as a synchronisation mechanism, to make sure that the
401 // encoder queue is not blocked before we start sending it frames.
402 void WaitUntilTaskQueueIsIdle() {
Niels Möllerc572ff32018-11-07 08:43:50 +0100403 rtc::Event event;
Yves Gerey665174f2018-06-19 15:03:05 +0200404 encoder_queue()->PostTask([&event] { event.Set(); });
kthelgason2fc52542017-03-03 00:24:41 -0800405 ASSERT_TRUE(event.Wait(5000));
406 }
407
Henrik Boström91aa7322020-04-28 12:24:33 +0200408 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200409 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200410 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200411 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200412 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200413 event.Set();
414 });
415 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200416 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200418
Henrik Boström91aa7322020-04-28 12:24:33 +0200419 void TriggerCpuUnderuse() {
420 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200421 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200422 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200423 event.Set();
424 });
425 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200426 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200427 }
kthelgason876222f2016-11-29 01:44:11 -0800428
Henrik Boström91aa7322020-04-28 12:24:33 +0200429 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200430 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200431 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200432 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200433 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200434 event.Set();
435 });
436 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200437 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200438 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200441 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200442 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 event.Set();
444 });
445 ASSERT_TRUE(event.Wait(5000));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200446 time_controller_->AdvanceTime(TimeDelta::Millis(0));
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 }
448
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200449 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100450 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200451 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
452 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200453 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700454};
455
Noah Richards51db4212019-06-12 06:59:12 -0700456// Simulates simulcast behavior and makes highest stream resolutions divisible
457// by 4.
458class CroppingVideoStreamFactory
459 : public VideoEncoderConfig::VideoStreamFactoryInterface {
460 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200461 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700462
463 private:
464 std::vector<VideoStream> CreateEncoderStreams(
465 int width,
466 int height,
467 const VideoEncoderConfig& encoder_config) override {
468 std::vector<VideoStream> streams = test::CreateVideoStreams(
469 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700470 return streams;
471 }
Noah Richards51db4212019-06-12 06:59:12 -0700472};
473
sprangb1ca0732017-02-01 08:38:12 -0800474class AdaptingFrameForwarder : public test::FrameForwarder {
475 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200476 explicit AdaptingFrameForwarder(TimeController* time_controller)
477 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700478 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800479
480 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200481 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800482 adaptation_enabled_ = enabled;
483 }
484
asaperssonfab67072017-04-04 05:51:49 -0700485 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200486 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800487 return adaptation_enabled_;
488 }
489
Henrik Boström1124ed12021-02-25 10:30:39 +0100490 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
491 // the resolution or frame rate was different than it is currently. If
492 // something else is modified, such as encoder resolutions, but the resolution
493 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700494 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200495 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700496 return last_wants_;
497 }
498
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200499 absl::optional<int> last_sent_width() const { return last_width_; }
500 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800501
sprangb1ca0732017-02-01 08:38:12 -0800502 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200503 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
504 time_controller_->AdvanceTime(TimeDelta::Millis(0));
505
sprangb1ca0732017-02-01 08:38:12 -0800506 int cropped_width = 0;
507 int cropped_height = 0;
508 int out_width = 0;
509 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700510 if (adaption_enabled()) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 RTC_DLOG(INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
512 << "w=" << video_frame.width()
513 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700514 if (adapter_.AdaptFrameResolution(
515 video_frame.width(), video_frame.height(),
516 video_frame.timestamp_us() * 1000, &cropped_width,
517 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100518 VideoFrame adapted_frame =
519 VideoFrame::Builder()
520 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
521 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200522 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100523 .set_timestamp_ms(99)
524 .set_rotation(kVideoRotation_0)
525 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100526 if (video_frame.has_update_rect()) {
527 adapted_frame.set_update_rect(
528 video_frame.update_rect().ScaleWithFrame(
529 video_frame.width(), video_frame.height(), 0, 0,
530 video_frame.width(), video_frame.height(), out_width,
531 out_height));
532 }
sprangc5d62e22017-04-02 23:53:04 -0700533 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800534 last_width_.emplace(adapted_frame.width());
535 last_height_.emplace(adapted_frame.height());
536 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200537 last_width_ = absl::nullopt;
538 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700539 }
sprangb1ca0732017-02-01 08:38:12 -0800540 } else {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200541 RTC_DLOG(INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800542 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800543 last_width_.emplace(video_frame.width());
544 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800545 }
546 }
547
548 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
549 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200550 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100551 rtc::VideoSinkWants prev_wants = sink_wants_locked();
552 bool did_adapt =
553 prev_wants.max_pixel_count != wants.max_pixel_count ||
554 prev_wants.target_pixel_count != wants.target_pixel_count ||
555 prev_wants.max_framerate_fps != wants.max_framerate_fps;
556 if (did_adapt) {
557 last_wants_ = prev_wants;
558 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100559 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200560 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800561 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200562
563 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800564 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200565 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
566 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200567 absl::optional<int> last_width_;
568 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800569};
sprangc5d62e22017-04-02 23:53:04 -0700570
Niels Möller213618e2018-07-24 09:29:58 +0200571// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700572class MockableSendStatisticsProxy : public SendStatisticsProxy {
573 public:
574 MockableSendStatisticsProxy(Clock* clock,
575 const VideoSendStream::Config& config,
576 VideoEncoderConfig::ContentType content_type)
577 : SendStatisticsProxy(clock, config, content_type) {}
578
579 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200580 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700581 if (mock_stats_)
582 return *mock_stats_;
583 return SendStatisticsProxy::GetStats();
584 }
585
Niels Möller213618e2018-07-24 09:29:58 +0200586 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200587 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200588 if (mock_stats_)
589 return mock_stats_->input_frame_rate;
590 return SendStatisticsProxy::GetInputFrameRate();
591 }
sprangc5d62e22017-04-02 23:53:04 -0700592 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200593 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700594 mock_stats_.emplace(stats);
595 }
596
597 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700599 mock_stats_.reset();
600 }
601
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200602 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
603 on_frame_dropped_ = std::move(callback);
604 }
605
sprangc5d62e22017-04-02 23:53:04 -0700606 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607 void OnFrameDropped(DropReason reason) override {
608 SendStatisticsProxy::OnFrameDropped(reason);
609 if (on_frame_dropped_)
610 on_frame_dropped_(reason);
611 }
612
Markus Handella3765182020-07-08 13:13:32 +0200613 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200614 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200615 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700616};
617
philipel9b058032020-02-10 11:30:00 +0100618class MockEncoderSelector
619 : public VideoEncoderFactory::EncoderSelectorInterface {
620 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200621 MOCK_METHOD(void,
622 OnCurrentEncoder,
623 (const SdpVideoFormat& format),
624 (override));
625 MOCK_METHOD(absl::optional<SdpVideoFormat>,
626 OnAvailableBitrate,
627 (const DataRate& rate),
628 (override));
629 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100630};
631
perkj803d97f2016-11-01 11:45:46 -0700632} // namespace
633
mflodmancc3d4422017-08-03 08:27:51 -0700634class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700635 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200636 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700637
mflodmancc3d4422017-08-03 08:27:51 -0700638 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700639 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700640 codec_width_(320),
641 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200642 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200643 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200644 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700645 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200646 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700647 video_send_config_,
648 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200649 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700650
651 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700652 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700653 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200654 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800655 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200656 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200657 video_send_config_.rtp.payload_name = "FAKE";
658 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700659
Per512ecb32016-09-23 15:52:06 +0200660 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200661 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200662 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
663 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
664 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100665 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700666
Niels Möllerf1338562018-04-26 09:51:47 +0200667 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800668 }
669
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100670 void ConfigureEncoder(
671 VideoEncoderConfig video_encoder_config,
672 VideoStreamEncoder::BitrateAllocationCallbackType
673 allocation_callback_type =
674 VideoStreamEncoder::BitrateAllocationCallbackType::
675 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700676 if (video_stream_encoder_)
677 video_stream_encoder_->Stop();
678 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200679 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100680 video_send_config_.encoder_settings, allocation_callback_type));
mflodmancc3d4422017-08-03 08:27:51 -0700681 video_stream_encoder_->SetSink(&sink_, false /* rotation_applied */);
682 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700683 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
mflodmancc3d4422017-08-03 08:27:51 -0700684 video_stream_encoder_->SetStartBitrate(kTargetBitrateBps);
685 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200686 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700687 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800688 }
689
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100690 void ResetEncoder(const std::string& payload_name,
691 size_t num_streams,
692 size_t num_temporal_layers,
693 unsigned char num_spatial_layers,
694 bool screenshare,
695 VideoStreamEncoder::BitrateAllocationCallbackType
696 allocation_callback_type =
697 VideoStreamEncoder::BitrateAllocationCallbackType::
698 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200699 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800700
701 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200702 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
703 num_streams, &video_encoder_config);
704 for (auto& layer : video_encoder_config.simulcast_layers) {
705 layer.num_temporal_layers = num_temporal_layers;
706 layer.max_framerate = kDefaultFramerate;
707 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100708 video_encoder_config.max_bitrate_bps =
709 num_streams == 1 ? kTargetBitrateBps : kSimulcastTargetBitrateBps;
sprang4847ae62017-06-27 07:06:52 -0700710 video_encoder_config.content_type =
711 screenshare ? VideoEncoderConfig::ContentType::kScreen
712 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700713 if (payload_name == "VP9") {
714 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
715 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200716 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700717 video_encoder_config.encoder_specific_settings =
718 new rtc::RefCountedObject<
719 VideoEncoderConfig::Vp9EncoderSpecificSettings>(vp9_settings);
720 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100721 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700722 }
723
sprang57c2fff2017-01-16 06:24:02 -0800724 VideoFrame CreateFrame(int64_t ntp_time_ms,
725 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200726 return VideoFrame::Builder()
727 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
728 destruction_event, codec_width_, codec_height_))
729 .set_ntp_time_ms(ntp_time_ms)
730 .set_timestamp_ms(99)
731 .set_rotation(kVideoRotation_0)
732 .build();
perkj26091b12016-09-01 01:17:40 -0700733 }
734
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100735 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
736 rtc::Event* destruction_event,
737 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200738 return VideoFrame::Builder()
739 .set_video_frame_buffer(new rtc::RefCountedObject<TestBuffer>(
740 destruction_event, codec_width_, codec_height_))
741 .set_ntp_time_ms(ntp_time_ms)
742 .set_timestamp_ms(99)
743 .set_rotation(kVideoRotation_0)
744 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
745 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100746 }
747
sprang57c2fff2017-01-16 06:24:02 -0800748 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200749 return VideoFrame::Builder()
750 .set_video_frame_buffer(
751 new rtc::RefCountedObject<TestBuffer>(nullptr, width, height))
752 .set_ntp_time_ms(ntp_time_ms)
753 .set_timestamp_ms(ntp_time_ms)
754 .set_rotation(kVideoRotation_0)
755 .build();
perkj803d97f2016-11-01 11:45:46 -0700756 }
757
Evan Shrubsole895556e2020-10-05 09:15:13 +0200758 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200759 return VideoFrame::Builder()
760 .set_video_frame_buffer(NV12Buffer::Create(width, height))
761 .set_ntp_time_ms(ntp_time_ms)
762 .set_timestamp_ms(ntp_time_ms)
763 .set_rotation(kVideoRotation_0)
764 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200765 }
766
Noah Richards51db4212019-06-12 06:59:12 -0700767 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
768 rtc::Event* destruction_event,
769 int width,
770 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200771 return VideoFrame::Builder()
772 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNativeBuffer>(
773 destruction_event, width, height))
774 .set_ntp_time_ms(ntp_time_ms)
775 .set_timestamp_ms(99)
776 .set_rotation(kVideoRotation_0)
777 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700778 }
779
Evan Shrubsole895556e2020-10-05 09:15:13 +0200780 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
781 rtc::Event* destruction_event,
782 int width,
783 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200784 return VideoFrame::Builder()
785 .set_video_frame_buffer(new rtc::RefCountedObject<FakeNV12NativeBuffer>(
786 destruction_event, width, height))
787 .set_ntp_time_ms(ntp_time_ms)
788 .set_timestamp_ms(99)
789 .set_rotation(kVideoRotation_0)
790 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200791 }
792
Noah Richards51db4212019-06-12 06:59:12 -0700793 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
794 rtc::Event* destruction_event) const {
795 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
796 codec_height_);
797 }
798
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100799 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +0100801 DataRate::BitsPerSec(kTargetBitrateBps),
802 DataRate::BitsPerSec(kTargetBitrateBps),
803 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100804
805 video_source_.IncomingCapturedFrame(
806 CreateFrame(1, codec_width_, codec_height_));
807 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200808 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100809 }
810
sprang4847ae62017-06-27 07:06:52 -0700811 void WaitForEncodedFrame(int64_t expected_ntp_time) {
812 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700814 }
815
816 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
817 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200818 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700819 return ok;
820 }
821
822 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
823 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700825 }
826
827 void ExpectDroppedFrame() {
828 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200829 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700830 }
831
832 bool WaitForFrame(int64_t timeout_ms) {
833 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700835 return ok;
836 }
837
perkj26091b12016-09-01 01:17:40 -0700838 class TestEncoder : public test::FakeEncoder {
839 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200840 explicit TestEncoder(TimeController* time_controller)
841 : FakeEncoder(time_controller->GetClock()),
842 time_controller_(time_controller) {
843 RTC_DCHECK(time_controller_);
844 }
perkj26091b12016-09-01 01:17:40 -0700845
asaperssonfab67072017-04-04 05:51:49 -0700846 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +0200847 MutexLock lock(&mutex_);
perkjfa10b552016-10-02 23:45:26 -0700848 return config_;
849 }
850
851 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200852 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700853 block_next_encode_ = true;
854 }
855
Erik Språngaed30702018-11-05 12:57:17 +0100856 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200857 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200858 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100859 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100860 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100861 info.scaling_settings = VideoEncoder::ScalingSettings(
862 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100863 }
864 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100865 for (int i = 0; i < kMaxSpatialLayers; ++i) {
866 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100867 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100868 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100869 for (int tid = 0; tid < num_layers; ++tid)
870 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100871 }
872 }
Erik Språngaed30702018-11-05 12:57:17 +0100873 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200874
875 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100876 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200877 info.apply_alignment_to_all_simulcast_layers =
878 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200879 info.preferred_pixel_formats = preferred_pixel_formats_;
Erik Språngaed30702018-11-05 12:57:17 +0100880 return info;
kthelgason876222f2016-11-29 01:44:11 -0800881 }
882
Erik Språngb7cb7b52019-02-26 15:52:33 +0100883 int32_t RegisterEncodeCompleteCallback(
884 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200885 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100886 encoded_image_callback_ = callback;
887 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
888 }
889
perkjfa10b552016-10-02 23:45:26 -0700890 void ContinueEncode() { continue_encode_event_.Set(); }
891
892 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
893 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200894 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700895 EXPECT_EQ(timestamp_, timestamp);
896 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
897 }
898
kthelgason2fc52542017-03-03 00:24:41 -0800899 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200900 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800901 quality_scaling_ = b;
902 }
kthelgasonad9010c2017-02-14 00:46:51 -0800903
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100904 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200905 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100906 requested_resolution_alignment_ = requested_resolution_alignment;
907 }
908
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200909 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
910 MutexLock lock(&local_mutex_);
911 apply_alignment_to_all_simulcast_layers_ = b;
912 }
913
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100914 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +0200915 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100916 is_hardware_accelerated_ = is_hardware_accelerated;
917 }
918
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100919 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
920 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +0200921 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100922 temporal_layers_supported_[spatial_idx] = supported;
923 }
924
Sergey Silkin6456e352019-07-08 17:56:40 +0200925 void SetResolutionBitrateLimits(
926 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +0200927 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +0200928 resolution_bitrate_limits_ = thresholds;
929 }
930
sprangfe627f32017-03-29 08:24:59 -0700931 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +0200932 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -0700933 force_init_encode_failed_ = force_failure;
934 }
935
Niels Möller6bb5ab92019-01-11 11:11:10 +0100936 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +0200937 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100938 rate_factor_ = rate_factor;
939 }
940
Erik Språngd7329ca2019-02-21 21:19:53 +0100941 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +0200942 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +0100943 return last_framerate_;
944 }
945
Erik Språngd7329ca2019-02-21 21:19:53 +0100946 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +0200947 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100948 return last_update_rect_;
949 }
950
Niels Möller87e2d782019-03-07 10:18:23 +0100951 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +0200952 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100953 return last_frame_types_;
954 }
955
956 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +0100957 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +0100958 keyframe ? VideoFrameType::kVideoFrameKey
959 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +0100960 {
Markus Handella3765182020-07-08 13:13:32 +0200961 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100962 last_frame_types_ = frame_type;
963 }
Niels Möllerb859b322019-03-07 12:40:01 +0100964 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +0100965 }
966
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100967 void InjectEncodedImage(const EncodedImage& image,
968 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +0200969 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100970 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100971 }
972
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200973 void SetEncodedImageData(
974 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +0200975 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200976 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200977 }
978
Erik Språngd7329ca2019-02-21 21:19:53 +0100979 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +0200980 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +0100981 expect_null_frame_ = true;
982 }
983
Erik Språng5056af02019-09-02 15:53:11 +0200984 absl::optional<VideoEncoder::RateControlParameters>
985 GetAndResetLastRateControlSettings() {
986 auto settings = last_rate_control_settings_;
987 last_rate_control_settings_.reset();
988 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +0100989 }
990
Henrik Boström56db9ff2021-03-24 09:06:45 +0100991 int GetLastInputWidth() const {
992 MutexLock lock(&local_mutex_);
993 return last_input_width_;
994 }
995
996 int GetLastInputHeight() const {
997 MutexLock lock(&local_mutex_);
998 return last_input_height_;
999 }
1000
Evan Shrubsole895556e2020-10-05 09:15:13 +02001001 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1002 MutexLock lock(&local_mutex_);
1003 return last_input_pixel_format_;
1004 }
1005
Sergey Silkin5ee69672019-07-02 14:18:34 +02001006 int GetNumEncoderInitializations() const {
Markus Handella3765182020-07-08 13:13:32 +02001007 MutexLock lock(&local_mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001008 return num_encoder_initializations_;
1009 }
1010
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001011 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001012 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001013 return num_set_rates_;
1014 }
1015
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001016 VideoCodec video_codec() const {
1017 MutexLock lock(&local_mutex_);
1018 return video_codec_;
1019 }
1020
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001021 void SetPreferredPixelFormats(
1022 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1023 pixel_formats) {
1024 MutexLock lock(&local_mutex_);
1025 preferred_pixel_formats_ = std::move(pixel_formats);
1026 }
1027
perkjfa10b552016-10-02 23:45:26 -07001028 private:
perkj26091b12016-09-01 01:17:40 -07001029 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001030 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001031 bool block_encode;
1032 {
Markus Handella3765182020-07-08 13:13:32 +02001033 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001034 if (expect_null_frame_) {
1035 EXPECT_EQ(input_image.timestamp(), 0u);
1036 EXPECT_EQ(input_image.width(), 1);
1037 last_frame_types_ = *frame_types;
1038 expect_null_frame_ = false;
1039 } else {
1040 EXPECT_GT(input_image.timestamp(), timestamp_);
1041 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1042 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1043 }
perkj26091b12016-09-01 01:17:40 -07001044
1045 timestamp_ = input_image.timestamp();
1046 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001047 last_input_width_ = input_image.width();
1048 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001049 block_encode = block_next_encode_;
1050 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001051 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001052 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001053 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001054 }
Niels Möllerb859b322019-03-07 12:40:01 +01001055 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001056 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001057 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001058
perkj26091b12016-09-01 01:17:40 -07001059 return result;
1060 }
1061
Niels Möller08ae7ce2020-09-23 15:58:12 +02001062 CodecSpecificInfo EncodeHook(
1063 EncodedImage& encoded_image,
1064 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001065 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001066 {
1067 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001068 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001069 }
1070 MutexLock lock(&local_mutex_);
1071 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001072 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001073 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001074 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001075 }
1076
sprangfe627f32017-03-29 08:24:59 -07001077 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001078 const Settings& settings) override {
1079 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001080
Markus Handella3765182020-07-08 13:13:32 +02001081 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001082 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001083
1084 ++num_encoder_initializations_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001085 video_codec_ = *config;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001086
Erik Språng82fad3d2018-03-21 09:57:23 +01001087 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001088 // Simulate setting up temporal layers, in order to validate the life
1089 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001090 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001091 frame_buffer_controller_ =
1092 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001093 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001094 if (force_init_encode_failed_) {
1095 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001096 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001097 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001098
Erik Språngb7cb7b52019-02-26 15:52:33 +01001099 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001100 return res;
1101 }
1102
Erik Språngb7cb7b52019-02-26 15:52:33 +01001103 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001104 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001105 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1106 initialized_ = EncoderState::kUninitialized;
1107 return FakeEncoder::Release();
1108 }
1109
Erik Språng16cb8f52019-04-12 13:59:09 +02001110 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001112 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001113 VideoBitrateAllocation adjusted_rate_allocation;
1114 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1115 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001116 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001117 adjusted_rate_allocation.SetBitrate(
1118 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001119 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001120 rate_factor_));
1121 }
1122 }
1123 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001124 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001125 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001126 RateControlParameters adjusted_paramters = parameters;
1127 adjusted_paramters.bitrate = adjusted_rate_allocation;
1128 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001129 }
1130
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001131 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001132 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001133 enum class EncoderState {
1134 kUninitialized,
1135 kInitializationFailed,
1136 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001137 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1138 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001139 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001140 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1141 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1142 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1143 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1144 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1145 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001146 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1147 false;
Markus Handella3765182020-07-08 13:13:32 +02001148 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001149 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1150 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001151 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001152 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001153 absl::optional<bool>
1154 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001155 local_mutex_);
1156 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1157 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1158 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001159 absl::optional<VideoEncoder::RateControlParameters>
1160 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001161 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1162 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001163 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001164 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001165 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1166 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001167 NiceMock<MockFecControllerOverride> fec_controller_override_;
Markus Handella3765182020-07-08 13:13:32 +02001168 int num_encoder_initializations_ RTC_GUARDED_BY(local_mutex_) = 0;
Sergey Silkin6456e352019-07-08 17:56:40 +02001169 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001170 RTC_GUARDED_BY(local_mutex_);
1171 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001172 VideoCodec video_codec_ RTC_GUARDED_BY(local_mutex_);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001173 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1174 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001175 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1176 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001177 };
1178
mflodmancc3d4422017-08-03 08:27:51 -07001179 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001180 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001181 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1182 : time_controller_(time_controller), test_encoder_(test_encoder) {
1183 RTC_DCHECK(time_controller_);
1184 }
perkj26091b12016-09-01 01:17:40 -07001185
perkj26091b12016-09-01 01:17:40 -07001186 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001187 EXPECT_TRUE(
1188 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1189 }
1190
1191 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1192 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001193 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001194 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001195 return false;
perkj26091b12016-09-01 01:17:40 -07001196 {
Markus Handella3765182020-07-08 13:13:32 +02001197 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001198 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001199 }
1200 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001201 return true;
perkj26091b12016-09-01 01:17:40 -07001202 }
1203
sprangb1ca0732017-02-01 08:38:12 -08001204 void WaitForEncodedFrame(uint32_t expected_width,
1205 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001206 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001207 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001208 }
1209
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001210 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001211 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001212 uint32_t width = 0;
1213 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001214 {
Markus Handella3765182020-07-08 13:13:32 +02001215 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001216 width = last_width_;
1217 height = last_height_;
1218 }
1219 EXPECT_EQ(expected_height, height);
1220 EXPECT_EQ(expected_width, width);
1221 }
1222
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001223 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1224 VideoRotation rotation;
1225 {
Markus Handella3765182020-07-08 13:13:32 +02001226 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001227 rotation = last_rotation_;
1228 }
1229 EXPECT_EQ(expected_rotation, rotation);
1230 }
1231
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001232 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001233
sprangc5d62e22017-04-02 23:53:04 -07001234 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001235 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
1236 bool ret = encoded_frame_event_.Wait(timeout_ms);
1237 time_controller_->AdvanceTime(TimeDelta::Millis(0));
1238 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001239 }
1240
perkj26091b12016-09-01 01:17:40 -07001241 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001242 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001243 expect_frames_ = false;
1244 }
1245
asaperssonfab67072017-04-04 05:51:49 -07001246 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001247 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001248 return number_of_reconfigurations_;
1249 }
1250
asaperssonfab67072017-04-04 05:51:49 -07001251 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001252 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001253 return min_transmit_bitrate_bps_;
1254 }
1255
Erik Språngd7329ca2019-02-21 21:19:53 +01001256 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001257 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001258 num_expected_layers_ = num_layers;
1259 }
1260
Erik Språngb7cb7b52019-02-26 15:52:33 +01001261 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001262 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001263 return last_capture_time_ms_;
1264 }
1265
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001266 const EncodedImage& GetLastEncodedImage() {
1267 MutexLock lock(&mutex_);
1268 return last_encoded_image_;
1269 }
1270
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001271 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001272 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001273 return std::move(last_encoded_image_data_);
1274 }
1275
Per Kjellanderdcef6412020-10-07 15:09:05 +02001276 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1277 MutexLock lock(&mutex_);
1278 return last_bitrate_allocation_;
1279 }
1280
1281 int number_of_bitrate_allocations() const {
1282 MutexLock lock(&mutex_);
1283 return number_of_bitrate_allocations_;
1284 }
1285
Per Kjellandera9434842020-10-15 17:53:22 +02001286 VideoLayersAllocation GetLastVideoLayersAllocation() {
1287 MutexLock lock(&mutex_);
1288 return last_layers_allocation_;
1289 }
1290
1291 int number_of_layers_allocations() const {
1292 MutexLock lock(&mutex_);
1293 return number_of_layers_allocations_;
1294 }
1295
perkj26091b12016-09-01 01:17:40 -07001296 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001297 Result OnEncodedImage(
1298 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001299 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001300 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001301 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001302 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001303 last_encoded_image_data_ = std::vector<uint8_t>(
1304 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001305 uint32_t timestamp = encoded_image.Timestamp();
1306 if (last_timestamp_ != timestamp) {
1307 num_received_layers_ = 1;
1308 } else {
1309 ++num_received_layers_;
1310 }
1311 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001312 last_capture_time_ms_ = encoded_image.capture_time_ms_;
sprangb1ca0732017-02-01 08:38:12 -08001313 last_width_ = encoded_image._encodedWidth;
1314 last_height_ = encoded_image._encodedHeight;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001315 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001316 if (num_received_layers_ == num_expected_layers_) {
1317 encoded_frame_event_.Set();
1318 }
sprangb1ca0732017-02-01 08:38:12 -08001319 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001320 }
1321
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001322 void OnEncoderConfigurationChanged(
1323 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001324 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001325 VideoEncoderConfig::ContentType content_type,
1326 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001327 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001328 ++number_of_reconfigurations_;
1329 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1330 }
1331
Per Kjellanderdcef6412020-10-07 15:09:05 +02001332 void OnBitrateAllocationUpdated(
1333 const VideoBitrateAllocation& allocation) override {
1334 MutexLock lock(&mutex_);
1335 ++number_of_bitrate_allocations_;
1336 last_bitrate_allocation_ = allocation;
1337 }
1338
Per Kjellandera9434842020-10-15 17:53:22 +02001339 void OnVideoLayersAllocationUpdated(
1340 VideoLayersAllocation allocation) override {
1341 MutexLock lock(&mutex_);
1342 ++number_of_layers_allocations_;
1343 last_layers_allocation_ = allocation;
1344 rtc::StringBuilder log;
1345 for (const auto& layer : allocation.active_spatial_layers) {
1346 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1347 << "[";
1348 for (const auto target_bitrate :
1349 layer.target_bitrate_per_temporal_layer) {
1350 log << target_bitrate.kbps() << ",";
1351 }
1352 log << "]";
1353 }
1354 RTC_DLOG(INFO) << "OnVideoLayersAllocationUpdated " << log.str();
1355 }
1356
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001357 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001358 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001359 TestEncoder* test_encoder_;
1360 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001361 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001362 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001363 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001364 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001365 uint32_t last_height_ = 0;
1366 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001367 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001368 size_t num_expected_layers_ = 1;
1369 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001370 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001371 int number_of_reconfigurations_ = 0;
1372 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001373 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1374 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001375 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1376 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001377 };
1378
Sergey Silkin5ee69672019-07-02 14:18:34 +02001379 class VideoBitrateAllocatorProxyFactory
1380 : public VideoBitrateAllocatorFactory {
1381 public:
1382 VideoBitrateAllocatorProxyFactory()
1383 : bitrate_allocator_factory_(
1384 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1385
1386 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1387 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001388 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001389 codec_config_ = codec;
1390 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1391 }
1392
1393 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001394 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001395 return codec_config_;
1396 }
1397
1398 private:
1399 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1400
Markus Handella3765182020-07-08 13:13:32 +02001401 mutable Mutex mutex_;
1402 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001403 };
1404
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001405 Clock* clock() { return time_controller_.GetClock(); }
1406 void AdvanceTime(TimeDelta duration) {
1407 time_controller_.AdvanceTime(duration);
1408 }
1409
1410 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1411
1412 protected:
1413 virtual TaskQueueFactory* GetTaskQueueFactory() {
1414 return time_controller_.GetTaskQueueFactory();
1415 }
1416
1417 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001418 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001419 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001420 int codec_width_;
1421 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001422 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001423 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001424 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001425 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001426 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001427 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001428 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001429 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001430};
1431
mflodmancc3d4422017-08-03 08:27:51 -07001432TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001434 DataRate::BitsPerSec(kTargetBitrateBps),
1435 DataRate::BitsPerSec(kTargetBitrateBps),
1436 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001437 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001438 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001439 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001440 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001441 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001442}
1443
mflodmancc3d4422017-08-03 08:27:51 -07001444TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001445 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001446 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001447 // The encoder will cache up to one frame for a short duration. Adding two
1448 // frames means that the first frame will be dropped and the second frame will
1449 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001450 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001451 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001452 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
perkja49cbd32016-09-16 07:53:41 -07001453 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001454
Henrik Boström381d1092020-05-12 18:49:07 +02001455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001456 DataRate::BitsPerSec(kTargetBitrateBps),
1457 DataRate::BitsPerSec(kTargetBitrateBps),
1458 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001459
Sebastian Janssona3177052018-04-10 13:05:49 +02001460 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001461 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001462 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1463
1464 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001465 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001466}
1467
mflodmancc3d4422017-08-03 08:27:51 -07001468TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001470 DataRate::BitsPerSec(kTargetBitrateBps),
1471 DataRate::BitsPerSec(kTargetBitrateBps),
1472 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001473 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001474 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001475
Henrik Boström381d1092020-05-12 18:49:07 +02001476 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1477 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
1478 0, 0, 0);
Sebastian Janssona3177052018-04-10 13:05:49 +02001479 // The encoder will cache up to one frame for a short duration. Adding two
1480 // frames means that the first frame will be dropped and the second frame will
1481 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001482 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001483 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001484
Henrik Boström381d1092020-05-12 18:49:07 +02001485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001486 DataRate::BitsPerSec(kTargetBitrateBps),
1487 DataRate::BitsPerSec(kTargetBitrateBps),
1488 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001489 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001490 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1491 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001492 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001493}
1494
mflodmancc3d4422017-08-03 08:27:51 -07001495TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001496 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001497 DataRate::BitsPerSec(kTargetBitrateBps),
1498 DataRate::BitsPerSec(kTargetBitrateBps),
1499 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001500 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001501 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001502
1503 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001504 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001505
perkja49cbd32016-09-16 07:53:41 -07001506 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001507 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001508 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001509}
1510
mflodmancc3d4422017-08-03 08:27:51 -07001511TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001513 DataRate::BitsPerSec(kTargetBitrateBps),
1514 DataRate::BitsPerSec(kTargetBitrateBps),
1515 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001516
perkja49cbd32016-09-16 07:53:41 -07001517 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001518 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001519
mflodmancc3d4422017-08-03 08:27:51 -07001520 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001521 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001522 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001523 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1524 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001525}
1526
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001527class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1528 public:
1529 VideoStreamEncoderBlockedTest() {}
1530
1531 TaskQueueFactory* GetTaskQueueFactory() override {
1532 return task_queue_factory_.get();
1533 }
1534
1535 private:
1536 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1537 CreateDefaultTaskQueueFactory();
1538};
1539
1540TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001541 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001542 DataRate::BitsPerSec(kTargetBitrateBps),
1543 DataRate::BitsPerSec(kTargetBitrateBps),
1544 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001545
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001546 int dropped_count = 0;
1547 stats_proxy_->SetDroppedFrameCallback(
1548 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1549 ++dropped_count;
1550 });
1551
perkj26091b12016-09-01 01:17:40 -07001552 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001553 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001554 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001555 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1556 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001557 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1558 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001559 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001560 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001561
mflodmancc3d4422017-08-03 08:27:51 -07001562 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001563
1564 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001565}
1566
Henrik Boström56db9ff2021-03-24 09:06:45 +01001567TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001569 DataRate::BitsPerSec(kTargetBitrateBps),
1570 DataRate::BitsPerSec(kTargetBitrateBps),
1571 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001572
1573 rtc::Event frame_destroyed_event;
1574 video_source_.IncomingCapturedFrame(
1575 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001576 WaitForEncodedFrame(1);
1577 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1578 fake_encoder_.GetLastInputPixelFormat());
1579 EXPECT_EQ(fake_encoder_.codec_config().width,
1580 fake_encoder_.GetLastInputWidth());
1581 EXPECT_EQ(fake_encoder_.codec_config().height,
1582 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001583 video_stream_encoder_->Stop();
1584}
1585
Henrik Boström56db9ff2021-03-24 09:06:45 +01001586TEST_F(VideoStreamEncoderTest,
1587 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001588 // Use the cropping factory.
1589 video_encoder_config_.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02001590 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001591 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1592 kMaxPayloadLength);
1593 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1594
1595 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001597 DataRate::BitsPerSec(kTargetBitrateBps),
1598 DataRate::BitsPerSec(kTargetBitrateBps),
1599 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001600 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1601 WaitForEncodedFrame(1);
1602 // The encoder will have been configured once.
1603 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1604 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1605 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1606
1607 // Now send in a fake frame that needs to be cropped as the width/height
1608 // aren't divisible by 4 (see CreateEncoderStreams above).
1609 rtc::Event frame_destroyed_event;
1610 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1611 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001612 WaitForEncodedFrame(2);
1613 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1614 fake_encoder_.GetLastInputPixelFormat());
1615 EXPECT_EQ(fake_encoder_.codec_config().width,
1616 fake_encoder_.GetLastInputWidth());
1617 EXPECT_EQ(fake_encoder_.codec_config().height,
1618 fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001619 video_stream_encoder_->Stop();
1620}
1621
Evan Shrubsole895556e2020-10-05 09:15:13 +02001622TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1624 DataRate::BitsPerSec(kTargetBitrateBps),
1625 DataRate::BitsPerSec(kTargetBitrateBps),
1626 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1627
1628 video_source_.IncomingCapturedFrame(
1629 CreateNV12Frame(1, codec_width_, codec_height_));
1630 WaitForEncodedFrame(1);
1631 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1632 fake_encoder_.GetLastInputPixelFormat());
1633 video_stream_encoder_->Stop();
1634}
1635
Henrik Boström56db9ff2021-03-24 09:06:45 +01001636TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1638 DataRate::BitsPerSec(kTargetBitrateBps),
1639 DataRate::BitsPerSec(kTargetBitrateBps),
1640 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1641
1642 fake_encoder_.SetPreferredPixelFormats({});
1643
1644 rtc::Event frame_destroyed_event;
1645 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1646 1, &frame_destroyed_event, codec_width_, codec_height_));
1647 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001648 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001649 fake_encoder_.GetLastInputPixelFormat());
1650 video_stream_encoder_->Stop();
1651}
1652
Henrik Boström56db9ff2021-03-24 09:06:45 +01001653TEST_F(VideoStreamEncoderTest,
1654 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001655 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1656 DataRate::BitsPerSec(kTargetBitrateBps),
1657 DataRate::BitsPerSec(kTargetBitrateBps),
1658 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1659
1660 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1661
1662 rtc::Event frame_destroyed_event;
1663 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1664 1, &frame_destroyed_event, codec_width_, codec_height_));
1665 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001666 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001667 fake_encoder_.GetLastInputPixelFormat());
1668 video_stream_encoder_->Stop();
1669}
1670
Henrik Boström56db9ff2021-03-24 09:06:45 +01001671TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1673 DataRate::BitsPerSec(kTargetBitrateBps),
1674 DataRate::BitsPerSec(kTargetBitrateBps),
1675 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1676
1677 // Fake NV12 native frame does not allow mapping to I444.
1678 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1679
1680 rtc::Event frame_destroyed_event;
1681 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1682 1, &frame_destroyed_event, codec_width_, codec_height_));
1683 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001684 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001685 fake_encoder_.GetLastInputPixelFormat());
1686 video_stream_encoder_->Stop();
1687}
1688
Henrik Boström56db9ff2021-03-24 09:06:45 +01001689TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
1691 DataRate::BitsPerSec(kTargetBitrateBps),
1692 DataRate::BitsPerSec(kTargetBitrateBps),
1693 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
1694
1695 rtc::Event frame_destroyed_event;
1696 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1697 1, &frame_destroyed_event, codec_width_, codec_height_));
1698 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001699 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001700 fake_encoder_.GetLastInputPixelFormat());
1701 video_stream_encoder_->Stop();
1702}
1703
Ying Wang9b881ab2020-02-07 14:29:32 +01001704TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001705 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001706 DataRate::BitsPerSec(kTargetBitrateBps),
1707 DataRate::BitsPerSec(kTargetBitrateBps),
1708 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001709 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1710 WaitForEncodedFrame(1);
1711
Henrik Boström381d1092020-05-12 18:49:07 +02001712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001713 DataRate::BitsPerSec(kTargetBitrateBps),
1714 DataRate::BitsPerSec(kTargetBitrateBps),
1715 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001716 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1717 // frames. Adding two frames means that the first frame will be dropped and
1718 // the second frame will be sent to the encoder.
1719 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1720 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1721 WaitForEncodedFrame(3);
1722 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1723 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1724 WaitForEncodedFrame(5);
1725 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1726 video_stream_encoder_->Stop();
1727}
1728
mflodmancc3d4422017-08-03 08:27:51 -07001729TEST_F(VideoStreamEncoderTest,
1730 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001732 DataRate::BitsPerSec(kTargetBitrateBps),
1733 DataRate::BitsPerSec(kTargetBitrateBps),
1734 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001735 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001736
1737 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001738 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001739 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001740 // The encoder will have been configured once when the first frame is
1741 // received.
1742 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001743
1744 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001745 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001746 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001747 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001748 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001749
1750 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001751 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001752 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001753 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001754 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001755
mflodmancc3d4422017-08-03 08:27:51 -07001756 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001757}
1758
mflodmancc3d4422017-08-03 08:27:51 -07001759TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001761 DataRate::BitsPerSec(kTargetBitrateBps),
1762 DataRate::BitsPerSec(kTargetBitrateBps),
1763 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001764
1765 // Capture a frame and wait for it to synchronize with the encoder thread.
1766 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001767 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001768 // The encoder will have been configured once.
1769 EXPECT_EQ(1, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001770 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1771 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
1772
1773 codec_width_ *= 2;
1774 codec_height_ *= 2;
1775 // Capture a frame with a higher resolution and wait for it to synchronize
1776 // with the encoder thread.
1777 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001778 WaitForEncodedFrame(2);
perkjfa10b552016-10-02 23:45:26 -07001779 EXPECT_EQ(codec_width_, fake_encoder_.codec_config().width);
1780 EXPECT_EQ(codec_height_, fake_encoder_.codec_config().height);
Per21d45d22016-10-30 21:37:57 +01001781 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001782
mflodmancc3d4422017-08-03 08:27:51 -07001783 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001784}
1785
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001786TEST_F(VideoStreamEncoderTest,
1787 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001789 DataRate::BitsPerSec(kTargetBitrateBps),
1790 DataRate::BitsPerSec(kTargetBitrateBps),
1791 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001792
1793 // Capture a frame and wait for it to synchronize with the encoder thread.
1794 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1795 WaitForEncodedFrame(1);
1796
1797 VideoEncoderConfig video_encoder_config;
1798 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1799 // Changing the max payload data length recreates encoder.
1800 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1801 kMaxPayloadLength / 2);
1802
1803 // Capture a frame and wait for it to synchronize with the encoder thread.
1804 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1805 WaitForEncodedFrame(2);
1806 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1807
1808 video_stream_encoder_->Stop();
1809}
1810
Sergey Silkin5ee69672019-07-02 14:18:34 +02001811TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001813 DataRate::BitsPerSec(kTargetBitrateBps),
1814 DataRate::BitsPerSec(kTargetBitrateBps),
1815 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001816
1817 VideoEncoderConfig video_encoder_config;
1818 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1819 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
1820 video_stream_encoder_->SetStartBitrate(kStartBitrateBps);
1821 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1822 kMaxPayloadLength);
1823
1824 // Capture a frame and wait for it to synchronize with the encoder thread.
1825 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1826 WaitForEncodedFrame(1);
1827 // The encoder will have been configured once when the first frame is
1828 // received.
1829 EXPECT_EQ(1, sink_.number_of_reconfigurations());
1830 EXPECT_EQ(kTargetBitrateBps,
1831 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1832 EXPECT_EQ(kStartBitrateBps,
1833 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1834
Sergey Silkin6456e352019-07-08 17:56:40 +02001835 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1836 &video_encoder_config); //???
Sergey Silkin5ee69672019-07-02 14:18:34 +02001837 video_encoder_config.max_bitrate_bps = kTargetBitrateBps * 2;
1838 video_stream_encoder_->SetStartBitrate(kStartBitrateBps * 2);
1839 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1840 kMaxPayloadLength);
1841
1842 // Capture a frame and wait for it to synchronize with the encoder thread.
1843 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1844 WaitForEncodedFrame(2);
1845 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1846 // Bitrate limits have changed - rate allocator should be reconfigured,
1847 // encoder should not be reconfigured.
1848 EXPECT_EQ(kTargetBitrateBps * 2,
1849 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1850 EXPECT_EQ(kStartBitrateBps * 2,
1851 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1852 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
1853
1854 video_stream_encoder_->Stop();
1855}
1856
Sergey Silkin6456e352019-07-08 17:56:40 +02001857TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001858 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001860 DataRate::BitsPerSec(kTargetBitrateBps),
1861 DataRate::BitsPerSec(kTargetBitrateBps),
1862 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001863
Sergey Silkincd02eba2020-01-20 14:48:40 +01001864 const uint32_t kMinEncBitrateKbps = 100;
1865 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001866 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001867 /*frame_size_pixels=*/codec_width_ * codec_height_,
1868 /*min_start_bitrate_bps=*/0,
1869 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1870 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001871 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1872
Sergey Silkincd02eba2020-01-20 14:48:40 +01001873 VideoEncoderConfig video_encoder_config;
1874 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1875 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1876 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1877 (kMinEncBitrateKbps + 1) * 1000;
1878 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1879 kMaxPayloadLength);
1880
1881 // When both encoder and app provide bitrate limits, the intersection of
1882 // provided sets should be used.
1883 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1884 WaitForEncodedFrame(1);
1885 EXPECT_EQ(kMaxEncBitrateKbps,
1886 bitrate_allocator_factory_.codec_config().maxBitrate);
1887 EXPECT_EQ(kMinEncBitrateKbps + 1,
1888 bitrate_allocator_factory_.codec_config().minBitrate);
1889
1890 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1891 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1892 (kMinEncBitrateKbps - 1) * 1000;
1893 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1894 kMaxPayloadLength);
1895 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001896 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001897 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001898 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001899 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 bitrate_allocator_factory_.codec_config().minBitrate);
1901
Sergey Silkincd02eba2020-01-20 14:48:40 +01001902 video_stream_encoder_->Stop();
1903}
1904
1905TEST_F(VideoStreamEncoderTest,
1906 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001908 DataRate::BitsPerSec(kTargetBitrateBps),
1909 DataRate::BitsPerSec(kTargetBitrateBps),
1910 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001911
1912 const uint32_t kMinAppBitrateKbps = 100;
1913 const uint32_t kMaxAppBitrateKbps = 200;
1914 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1915 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1916 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1917 /*frame_size_pixels=*/codec_width_ * codec_height_,
1918 /*min_start_bitrate_bps=*/0,
1919 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1920 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1921 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1922
1923 VideoEncoderConfig video_encoder_config;
1924 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1925 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1926 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1927 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001928 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1929 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001930
Sergey Silkincd02eba2020-01-20 14:48:40 +01001931 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1932 WaitForEncodedFrame(1);
1933 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001934 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001935 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001936 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001937
1938 video_stream_encoder_->Stop();
1939}
1940
1941TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001942 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01001944 DataRate::BitsPerSec(kTargetBitrateBps),
1945 DataRate::BitsPerSec(kTargetBitrateBps),
1946 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001947
1948 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001949 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001950 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001951 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001952 fake_encoder_.SetResolutionBitrateLimits(
1953 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1954
1955 VideoEncoderConfig video_encoder_config;
1956 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1957 video_encoder_config.max_bitrate_bps = 0;
1958 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1959 kMaxPayloadLength);
1960
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001961 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001962 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1963 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1965 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001966 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1967 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1968
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001969 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001970 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1971 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001972 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1973 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001974 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1975 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1976
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001977 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001978 // encoder for 360p should be used.
1979 video_source_.IncomingCapturedFrame(
1980 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
1981 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001982 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1983 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001984 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1985 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1986
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001987 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02001988 // ignored.
1989 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
1990 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1992 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001993 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1994 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1996 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001997 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1998 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1999
2000 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2001 // for 270p should be used.
2002 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2003 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002006 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2007 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2008
2009 video_stream_encoder_->Stop();
2010}
2011
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002012TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002014 DataRate::BitsPerSec(kTargetBitrateBps),
2015 DataRate::BitsPerSec(kTargetBitrateBps),
2016 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017
2018 VideoEncoderConfig video_encoder_config;
2019 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2020 video_encoder_config.max_bitrate_bps = 0;
2021 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2022 kMaxPayloadLength);
2023
2024 // Encode 720p frame to get the default encoder target bitrate.
2025 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2026 WaitForEncodedFrame(1);
2027 const uint32_t kDefaultTargetBitrateFor720pKbps =
2028 bitrate_allocator_factory_.codec_config()
2029 .simulcastStream[0]
2030 .targetBitrate;
2031
2032 // Set the max recommended encoder bitrate to something lower than the default
2033 // target bitrate.
2034 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2035 1280 * 720, 10 * 1000, 10 * 1000,
2036 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2037 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2038
2039 // Change resolution to trigger encoder reinitialization.
2040 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2041 WaitForEncodedFrame(2);
2042 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2043 WaitForEncodedFrame(3);
2044
2045 // Ensure the target bitrate is capped by the max bitrate.
2046 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2047 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2048 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2049 .simulcastStream[0]
2050 .targetBitrate *
2051 1000,
2052 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2053
2054 video_stream_encoder_->Stop();
2055}
2056
Åsa Perssona7e34d32021-01-20 15:36:13 +01002057TEST_F(VideoStreamEncoderTest,
2058 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2059 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2060 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2061 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2062 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2063 fake_encoder_.SetResolutionBitrateLimits(
2064 {kEncoderLimits270p, kEncoderLimits360p});
2065
2066 // Two streams, highest stream active.
2067 VideoEncoderConfig config;
2068 const int kNumStreams = 2;
2069 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2070 config.max_bitrate_bps = 0;
2071 config.simulcast_layers[0].active = false;
2072 config.simulcast_layers[1].active = true;
2073 config.video_stream_factory =
2074 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2075 "VP8", /*max qp*/ 56, /*screencast*/ false,
2076 /*screenshare enabled*/ false);
2077 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2078
2079 // The encoder bitrate limits for 270p should be used.
2080 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2081 EXPECT_FALSE(WaitForFrame(1000));
2082 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2083 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2084 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2085 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2086 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2087
2088 // The encoder bitrate limits for 360p should be used.
2089 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2090 EXPECT_FALSE(WaitForFrame(1000));
2091 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2092 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2093 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2094 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2095
2096 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2097 video_source_.IncomingCapturedFrame(
2098 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2099 EXPECT_FALSE(WaitForFrame(1000));
2100 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2101 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2102 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2103 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2104
2105 // Resolution higher than 360p. Encoder limits should be ignored.
2106 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2107 EXPECT_FALSE(WaitForFrame(1000));
2108 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2109 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2110 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2111 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2112 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2113 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2114 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2115 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2116
2117 // Resolution lower than 270p. The encoder limits for 270p should be used.
2118 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2119 EXPECT_FALSE(WaitForFrame(1000));
2120 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2121 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2122 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2123 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2124
2125 video_stream_encoder_->Stop();
2126}
2127
2128TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002129 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2130 // Two streams, highest stream active.
2131 VideoEncoderConfig config;
2132 const int kNumStreams = 2;
2133 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2134 config.max_bitrate_bps = 0;
2135 config.simulcast_layers[0].active = false;
2136 config.simulcast_layers[1].active = true;
2137 config.video_stream_factory =
2138 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2139 "VP8", /*max qp*/ 56, /*screencast*/ false,
2140 /*screenshare enabled*/ false);
2141 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2142
2143 // Default bitrate limits for 270p should be used.
2144 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2145 kDefaultLimits270p =
2146 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002147 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002148 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2149 EXPECT_FALSE(WaitForFrame(1000));
2150 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2151 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
2152 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2153 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
2154 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2155
2156 // Default bitrate limits for 360p should be used.
2157 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2158 kDefaultLimits360p =
2159 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002160 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002161 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2162 EXPECT_FALSE(WaitForFrame(1000));
2163 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2164 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2165 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2166 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2167
2168 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2169 video_source_.IncomingCapturedFrame(
2170 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2171 EXPECT_FALSE(WaitForFrame(1000));
2172 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
2173 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2174 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
2175 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2176
2177 // Default bitrate limits for 540p should be used.
2178 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2179 kDefaultLimits540p =
2180 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002181 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002182 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2183 EXPECT_FALSE(WaitForFrame(1000));
2184 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
2185 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2186 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
2187 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2188
2189 video_stream_encoder_->Stop();
2190}
2191
2192TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002193 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2194 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2195 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2196 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2197 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2198 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2199 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2200 fake_encoder_.SetResolutionBitrateLimits(
2201 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2202
2203 // Three streams, middle stream active.
2204 VideoEncoderConfig config;
2205 const int kNumStreams = 3;
2206 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2207 config.simulcast_layers[0].active = false;
2208 config.simulcast_layers[1].active = true;
2209 config.simulcast_layers[2].active = false;
2210 config.video_stream_factory =
2211 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2212 "VP8", /*max qp*/ 56, /*screencast*/ false,
2213 /*screenshare enabled*/ false);
2214 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2215
2216 // The encoder bitrate limits for 360p should be used.
2217 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2218 EXPECT_FALSE(WaitForFrame(1000));
2219 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2220 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2221 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2222 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
2223 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2224
2225 // The encoder bitrate limits for 270p should be used.
2226 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2227 EXPECT_FALSE(WaitForFrame(1000));
2228 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2229 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2230 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2231 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2232
2233 video_stream_encoder_->Stop();
2234}
2235
2236TEST_F(VideoStreamEncoderTest,
2237 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2238 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2239 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2240 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2241 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2242 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2243 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2244 fake_encoder_.SetResolutionBitrateLimits(
2245 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2246
2247 // Three streams, lowest stream active.
2248 VideoEncoderConfig config;
2249 const int kNumStreams = 3;
2250 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2251 config.simulcast_layers[0].active = true;
2252 config.simulcast_layers[1].active = false;
2253 config.simulcast_layers[2].active = false;
2254 config.video_stream_factory =
2255 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2256 "VP8", /*max qp*/ 56, /*screencast*/ false,
2257 /*screenshare enabled*/ false);
2258 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2259
2260 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2261 // on lowest stream, limits for 270p should not be used
2262 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2263 EXPECT_FALSE(WaitForFrame(1000));
2264 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2265 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2266 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2267 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2268 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2269
2270 video_stream_encoder_->Stop();
2271}
2272
2273TEST_F(VideoStreamEncoderTest,
2274 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2275 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2276 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2277 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2278 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2279 fake_encoder_.SetResolutionBitrateLimits(
2280 {kEncoderLimits270p, kEncoderLimits360p});
2281 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2282
2283 // Two streams, highest stream active.
2284 VideoEncoderConfig config;
2285 const int kNumStreams = 2;
2286 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2287 config.simulcast_layers[0].active = false;
2288 config.simulcast_layers[1].active = true;
2289 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2290 config.video_stream_factory =
2291 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2292 "VP8", /*max qp*/ 56, /*screencast*/ false,
2293 /*screenshare enabled*/ false);
2294 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2295
2296 // The encoder bitrate limits for 270p should be used.
2297 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2298 EXPECT_FALSE(WaitForFrame(1000));
2299 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, kNumStreams);
2300 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
2301 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2302 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
2303 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2304
2305 // The max configured bitrate is less than the encoder limit for 360p.
2306 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2307 EXPECT_FALSE(WaitForFrame(1000));
2308 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
2309 fake_encoder_.video_codec().simulcastStream[1].minBitrate * 1000);
2310 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
2311 fake_encoder_.video_codec().simulcastStream[1].maxBitrate * 1000);
2312
2313 video_stream_encoder_->Stop();
2314}
2315
mflodmancc3d4422017-08-03 08:27:51 -07002316TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002317 EXPECT_TRUE(video_source_.has_sinks());
2318 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002319 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002320 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002321 EXPECT_FALSE(video_source_.has_sinks());
2322 EXPECT_TRUE(new_video_source.has_sinks());
2323
mflodmancc3d4422017-08-03 08:27:51 -07002324 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002325}
2326
mflodmancc3d4422017-08-03 08:27:51 -07002327TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002328 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002329 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002330 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002331 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002332}
2333
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002334class ResolutionAlignmentTest
2335 : public VideoStreamEncoderTest,
2336 public ::testing::WithParamInterface<
2337 ::testing::tuple<int, std::vector<double>>> {
2338 public:
2339 ResolutionAlignmentTest()
2340 : requested_alignment_(::testing::get<0>(GetParam())),
2341 scale_factors_(::testing::get<1>(GetParam())) {}
2342
2343 protected:
2344 const int requested_alignment_;
2345 const std::vector<double> scale_factors_;
2346};
2347
2348INSTANTIATE_TEST_SUITE_P(
2349 AlignmentAndScaleFactors,
2350 ResolutionAlignmentTest,
2351 ::testing::Combine(
2352 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2353 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2354 std::vector<double>{-1.0, -1.0},
2355 std::vector<double>{-1.0, -1.0, -1.0},
2356 std::vector<double>{4.0, 2.0, 1.0},
2357 std::vector<double>{9999.0, -1.0, 1.0},
2358 std::vector<double>{3.99, 2.01, 1.0},
2359 std::vector<double>{4.9, 1.7, 1.25},
2360 std::vector<double>{10.0, 4.0, 3.0},
2361 std::vector<double>{1.75, 3.5},
2362 std::vector<double>{1.5, 2.5},
2363 std::vector<double>{1.3, 1.0})));
2364
2365TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2366 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002367 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002368 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2369 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2370
2371 // Fill config with the scaling factor by which to reduce encoding size.
2372 const int num_streams = scale_factors_.size();
2373 VideoEncoderConfig config;
2374 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2375 for (int i = 0; i < num_streams; ++i) {
2376 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2377 }
2378 config.video_stream_factory =
2379 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
2380 "VP8", /*max qp*/ 56, /*screencast*/ false,
2381 /*screenshare enabled*/ false);
2382 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2383
Henrik Boström381d1092020-05-12 18:49:07 +02002384 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002385 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2386 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
2387 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
2388 // Wait for all layers before triggering event.
2389 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002390
2391 // On the 1st frame, we should have initialized the encoder and
2392 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002393 int64_t timestamp_ms = kFrameIntervalMs;
2394 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2395 WaitForEncodedFrame(timestamp_ms);
2396 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002397
2398 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2399 // (It's up the to the encoder to potentially drop the previous frame,
2400 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002401 timestamp_ms += kFrameIntervalMs;
2402 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2403 WaitForEncodedFrame(timestamp_ms);
2404 EXPECT_GE(fake_encoder_.GetNumEncoderInitializations(), 1);
2405
2406 VideoCodec codec = fake_encoder_.video_codec();
2407 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2408 // Frame size should be a multiple of the requested alignment.
2409 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2410 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2411 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2412 // Aspect ratio should match.
2413 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2414 codec.height * codec.simulcastStream[i].width);
2415 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002416
2417 video_stream_encoder_->Stop();
2418}
2419
Jonathan Yubc771b72017-12-08 17:04:29 -08002420TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2421 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002422 const int kWidth = 1280;
2423 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002424
2425 // We rely on the automatic resolution adaptation, but we handle framerate
2426 // adaptation manually by mocking the stats proxy.
2427 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002428
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002429 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002430 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002431 DataRate::BitsPerSec(kTargetBitrateBps),
2432 DataRate::BitsPerSec(kTargetBitrateBps),
2433 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002434 video_stream_encoder_->SetSource(&video_source_,
2435 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002436 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002437 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002438 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002439 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2440
Jonathan Yubc771b72017-12-08 17:04:29 -08002441 // Adapt down as far as possible.
2442 rtc::VideoSinkWants last_wants;
2443 int64_t t = 1;
2444 int loop_count = 0;
2445 do {
2446 ++loop_count;
2447 last_wants = video_source_.sink_wants();
2448
2449 // Simulate the framerate we've been asked to adapt to.
2450 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2451 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2452 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2453 mock_stats.input_frame_rate = fps;
2454 stats_proxy_->SetMockStats(mock_stats);
2455
2456 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2457 sink_.WaitForEncodedFrame(t);
2458 t += frame_interval_ms;
2459
mflodmancc3d4422017-08-03 08:27:51 -07002460 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002461 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002462 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002463 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2464 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002465 } while (video_source_.sink_wants().max_pixel_count <
2466 last_wants.max_pixel_count ||
2467 video_source_.sink_wants().max_framerate_fps <
2468 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002469
Jonathan Yubc771b72017-12-08 17:04:29 -08002470 // Verify that we've adapted all the way down.
2471 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002472 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002473 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2474 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002475 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002476 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2477 *video_source_.last_sent_height());
2478 EXPECT_EQ(kMinBalancedFramerateFps,
2479 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002480
Jonathan Yubc771b72017-12-08 17:04:29 -08002481 // Adapt back up the same number of times we adapted down.
2482 for (int i = 0; i < loop_count - 1; ++i) {
2483 last_wants = video_source_.sink_wants();
2484
2485 // Simulate the framerate we've been asked to adapt to.
2486 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2487 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2488 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2489 mock_stats.input_frame_rate = fps;
2490 stats_proxy_->SetMockStats(mock_stats);
2491
2492 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2493 sink_.WaitForEncodedFrame(t);
2494 t += frame_interval_ms;
2495
Henrik Boström91aa7322020-04-28 12:24:33 +02002496 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002497 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002498 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002499 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2500 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002501 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2502 last_wants.max_pixel_count ||
2503 video_source_.sink_wants().max_framerate_fps >
2504 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002505 }
2506
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002507 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002508 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002509 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002510 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2511 EXPECT_EQ((loop_count - 1) * 2,
2512 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002513
mflodmancc3d4422017-08-03 08:27:51 -07002514 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002515}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002516
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002517TEST_F(VideoStreamEncoderTest,
2518 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
2519 video_stream_encoder_->OnBitrateUpdated(
2520 DataRate::BitsPerSec(kTargetBitrateBps),
2521 DataRate::BitsPerSec(kTargetBitrateBps),
2522 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002523 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002524
2525 const int kFrameWidth = 1280;
2526 const int kFrameHeight = 720;
2527
2528 int64_t ntp_time = kFrameIntervalMs;
2529
2530 // Force an input frame rate to be available, or the adaptation call won't
2531 // know what framerate to adapt form.
2532 const int kInputFps = 30;
2533 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2534 stats.input_frame_rate = kInputFps;
2535 stats_proxy_->SetMockStats(stats);
2536
2537 video_source_.set_adaptation_enabled(true);
2538 video_stream_encoder_->SetSource(
2539 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002540 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002541 video_source_.IncomingCapturedFrame(
2542 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2543 sink_.WaitForEncodedFrame(ntp_time);
2544 ntp_time += kFrameIntervalMs;
2545
2546 // Trigger CPU overuse.
2547 video_stream_encoder_->TriggerCpuOveruse();
2548 video_source_.IncomingCapturedFrame(
2549 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2550 sink_.WaitForEncodedFrame(ntp_time);
2551 ntp_time += kFrameIntervalMs;
2552
2553 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2554 EXPECT_EQ(std::numeric_limits<int>::max(),
2555 video_source_.sink_wants().max_pixel_count);
2556 // Some framerate constraint should be set.
2557 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2558 EXPECT_LT(restricted_fps, kInputFps);
2559 video_source_.IncomingCapturedFrame(
2560 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2561 sink_.WaitForEncodedFrame(ntp_time);
2562 ntp_time += 100;
2563
Henrik Boström2671dac2020-05-19 16:29:09 +02002564 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002565 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2566 // Give the encoder queue time to process the change in degradation preference
2567 // by waiting for an encoded frame.
2568 video_source_.IncomingCapturedFrame(
2569 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2570 sink_.WaitForEncodedFrame(ntp_time);
2571 ntp_time += kFrameIntervalMs;
2572
2573 video_stream_encoder_->TriggerQualityLow();
2574 video_source_.IncomingCapturedFrame(
2575 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2576 sink_.WaitForEncodedFrame(ntp_time);
2577 ntp_time += kFrameIntervalMs;
2578
2579 // Some resolution constraint should be set.
2580 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2581 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2582 kFrameWidth * kFrameHeight);
2583 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2584
2585 int pixel_count = video_source_.sink_wants().max_pixel_count;
2586 // Triggering a CPU underuse should not change the sink wants since it has
2587 // not been overused for resolution since we changed degradation preference.
2588 video_stream_encoder_->TriggerCpuUnderuse();
2589 video_source_.IncomingCapturedFrame(
2590 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2591 sink_.WaitForEncodedFrame(ntp_time);
2592 ntp_time += kFrameIntervalMs;
2593 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2594 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2595
Evan Shrubsole64469032020-06-11 10:45:29 +02002596 // Change the degradation preference back. CPU underuse should not adapt since
2597 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002598 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002599 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2600 video_source_.IncomingCapturedFrame(
2601 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2602 sink_.WaitForEncodedFrame(ntp_time);
2603 ntp_time += 100;
2604 // Resolution adaptations is gone after changing degradation preference.
2605 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2606 EXPECT_EQ(std::numeric_limits<int>::max(),
2607 video_source_.sink_wants().max_pixel_count);
2608 // The fps adaptation from above is now back.
2609 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2610
2611 // Trigger CPU underuse.
2612 video_stream_encoder_->TriggerCpuUnderuse();
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002617 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2618
2619 // Trigger QP underuse, fps should return to normal.
2620 video_stream_encoder_->TriggerQualityHigh();
2621 video_source_.IncomingCapturedFrame(
2622 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2623 sink_.WaitForEncodedFrame(ntp_time);
2624 ntp_time += kFrameIntervalMs;
2625 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002626
2627 video_stream_encoder_->Stop();
2628}
2629
mflodmancc3d4422017-08-03 08:27:51 -07002630TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002632 DataRate::BitsPerSec(kTargetBitrateBps),
2633 DataRate::BitsPerSec(kTargetBitrateBps),
2634 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002635 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002636
sprangc5d62e22017-04-02 23:53:04 -07002637 const int kFrameWidth = 1280;
2638 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002639
Åsa Persson8c1bf952018-09-13 10:42:19 +02002640 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002641
kthelgason5e13d412016-12-01 03:59:51 -08002642 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002643 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002644 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002645 frame_timestamp += kFrameIntervalMs;
2646
perkj803d97f2016-11-01 11:45:46 -07002647 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002648 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002649 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002650 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002651 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002652 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002653
asapersson0944a802017-04-07 00:57:58 -07002654 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002655 // wanted resolution.
2656 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2657 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2658 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002659 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002660
2661 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002662 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002663 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002664 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002665 // Give the encoder queue time to process the change in degradation preference
2666 // by waiting for an encoded frame.
2667 new_video_source.IncomingCapturedFrame(
2668 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2669 sink_.WaitForEncodedFrame(frame_timestamp);
2670 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002671 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002672 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002673
sprangc5d62e22017-04-02 23:53:04 -07002674 // Force an input frame rate to be available, or the adaptation call won't
2675 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002676 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002677 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002678 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002679 stats_proxy_->SetMockStats(stats);
2680
mflodmancc3d4422017-08-03 08:27:51 -07002681 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002682 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002683 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002684 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002685 frame_timestamp += kFrameIntervalMs;
2686
2687 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002688 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002689 EXPECT_EQ(std::numeric_limits<int>::max(),
2690 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002691 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002692
asapersson02465b82017-04-10 01:12:52 -07002693 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002694 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2695 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002696 // Give the encoder queue time to process the change in degradation preference
2697 // by waiting for an encoded frame.
2698 new_video_source.IncomingCapturedFrame(
2699 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2700 sink_.WaitForEncodedFrame(frame_timestamp);
2701 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002702 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002703
mflodmancc3d4422017-08-03 08:27:51 -07002704 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002705 new_video_source.IncomingCapturedFrame(
2706 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002707 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002708 frame_timestamp += kFrameIntervalMs;
2709
2710 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002711 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002712
2713 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002714 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002715 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002716 // Give the encoder queue time to process the change in degradation preference
2717 // by waiting for an encoded frame.
2718 new_video_source.IncomingCapturedFrame(
2719 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2720 sink_.WaitForEncodedFrame(frame_timestamp);
2721 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002722 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2723 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002724 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002725 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002726
2727 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002728 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002729 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002730 // Give the encoder queue time to process the change in degradation preference
2731 // by waiting for an encoded frame.
2732 new_video_source.IncomingCapturedFrame(
2733 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2734 sink_.WaitForEncodedFrame(frame_timestamp);
2735 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002736 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2737 EXPECT_EQ(std::numeric_limits<int>::max(),
2738 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002739 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002740
mflodmancc3d4422017-08-03 08:27:51 -07002741 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002742}
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002746 DataRate::BitsPerSec(kTargetBitrateBps),
2747 DataRate::BitsPerSec(kTargetBitrateBps),
2748 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002749
asaperssonfab67072017-04-04 05:51:49 -07002750 const int kWidth = 1280;
2751 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002752 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002753 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002754 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2755 EXPECT_FALSE(stats.bw_limited_resolution);
2756 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2757
2758 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002759 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002760 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002761 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002762
2763 stats = stats_proxy_->GetStats();
2764 EXPECT_TRUE(stats.bw_limited_resolution);
2765 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2766
2767 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002768 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002769 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002771
2772 stats = stats_proxy_->GetStats();
2773 EXPECT_FALSE(stats.bw_limited_resolution);
2774 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2775 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2776
mflodmancc3d4422017-08-03 08:27:51 -07002777 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002778}
2779
mflodmancc3d4422017-08-03 08:27:51 -07002780TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002782 DataRate::BitsPerSec(kTargetBitrateBps),
2783 DataRate::BitsPerSec(kTargetBitrateBps),
2784 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002785
2786 const int kWidth = 1280;
2787 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002788 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002789 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002790 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2791 EXPECT_FALSE(stats.cpu_limited_resolution);
2792 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2793
2794 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002796 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002797 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002798
2799 stats = stats_proxy_->GetStats();
2800 EXPECT_TRUE(stats.cpu_limited_resolution);
2801 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2802
2803 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002804 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002805 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002806 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002807
2808 stats = stats_proxy_->GetStats();
2809 EXPECT_FALSE(stats.cpu_limited_resolution);
2810 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002811 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002812
mflodmancc3d4422017-08-03 08:27:51 -07002813 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002814}
2815
mflodmancc3d4422017-08-03 08:27:51 -07002816TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002818 DataRate::BitsPerSec(kTargetBitrateBps),
2819 DataRate::BitsPerSec(kTargetBitrateBps),
2820 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002821
asaperssonfab67072017-04-04 05:51:49 -07002822 const int kWidth = 1280;
2823 const int kHeight = 720;
2824 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002825 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002826 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002827 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002828 EXPECT_FALSE(stats.cpu_limited_resolution);
2829 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2830
asaperssonfab67072017-04-04 05:51:49 -07002831 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002832 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002833 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002834 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002835 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002836 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002837 EXPECT_TRUE(stats.cpu_limited_resolution);
2838 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2839
2840 // Set new source with adaptation still enabled.
2841 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002842 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002843 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002844
asaperssonfab67072017-04-04 05:51:49 -07002845 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002846 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002847 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002848 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002849 EXPECT_TRUE(stats.cpu_limited_resolution);
2850 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2851
2852 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002853 video_stream_encoder_->SetSource(&new_video_source,
2854 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002855
asaperssonfab67072017-04-04 05:51:49 -07002856 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002857 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002858 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002859 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002860 EXPECT_FALSE(stats.cpu_limited_resolution);
2861 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2862
2863 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002864 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002865 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002866
asaperssonfab67072017-04-04 05:51:49 -07002867 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002868 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002869 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002870 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002871 EXPECT_TRUE(stats.cpu_limited_resolution);
2872 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2873
asaperssonfab67072017-04-04 05:51:49 -07002874 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002875 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002876 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002877 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002878 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002879 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002880 EXPECT_FALSE(stats.cpu_limited_resolution);
2881 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002882 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002883
mflodmancc3d4422017-08-03 08:27:51 -07002884 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002885}
2886
mflodmancc3d4422017-08-03 08:27:51 -07002887TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002889 DataRate::BitsPerSec(kTargetBitrateBps),
2890 DataRate::BitsPerSec(kTargetBitrateBps),
2891 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002892
asaperssonfab67072017-04-04 05:51:49 -07002893 const int kWidth = 1280;
2894 const int kHeight = 720;
2895 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002896 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002897 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002898 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002899 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002900 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002901
2902 // Set new source with adaptation still enabled.
2903 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002904 video_stream_encoder_->SetSource(&new_video_source,
2905 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002906
asaperssonfab67072017-04-04 05:51:49 -07002907 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002908 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002909 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002910 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002911 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002912 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002913
asaperssonfab67072017-04-04 05:51:49 -07002914 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002915 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002916 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002917 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002918 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002919 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002920 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002921 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002922
asaperssonfab67072017-04-04 05:51:49 -07002923 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002924 video_stream_encoder_->SetSource(&new_video_source,
2925 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002926
asaperssonfab67072017-04-04 05:51:49 -07002927 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002928 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002929 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002930 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002931 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002932 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002933
asapersson02465b82017-04-10 01:12:52 -07002934 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002936 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002937
asaperssonfab67072017-04-04 05:51:49 -07002938 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002939 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002940 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002941 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002942 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002943 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2944 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002945
mflodmancc3d4422017-08-03 08:27:51 -07002946 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002947}
2948
mflodmancc3d4422017-08-03 08:27:51 -07002949TEST_F(VideoStreamEncoderTest,
2950 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002951 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01002952 DataRate::BitsPerSec(kTargetBitrateBps),
2953 DataRate::BitsPerSec(kTargetBitrateBps),
2954 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002955
2956 const int kWidth = 1280;
2957 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002958 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002959 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002960 video_source_.IncomingCapturedFrame(
2961 CreateFrame(timestamp_ms, kWidth, kHeight));
2962 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002963 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2964 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2965 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2966
2967 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002968 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002969 timestamp_ms += kFrameIntervalMs;
2970 video_source_.IncomingCapturedFrame(
2971 CreateFrame(timestamp_ms, kWidth, kHeight));
2972 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002973 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2974 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2975 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2976
2977 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002978 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002979 timestamp_ms += kFrameIntervalMs;
2980 video_source_.IncomingCapturedFrame(
2981 CreateFrame(timestamp_ms, kWidth, kHeight));
2982 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002983 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2984 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2985 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2986
Niels Möller4db138e2018-04-19 09:04:13 +02002987 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002988 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002989
2990 VideoEncoderConfig video_encoder_config;
2991 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2992 // Make format different, to force recreation of encoder.
2993 video_encoder_config.video_format.parameters["foo"] = "foo";
2994 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002995 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002996 timestamp_ms += kFrameIntervalMs;
2997 video_source_.IncomingCapturedFrame(
2998 CreateFrame(timestamp_ms, kWidth, kHeight));
2999 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003000 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3001 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3002 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3003
mflodmancc3d4422017-08-03 08:27:51 -07003004 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003005}
3006
mflodmancc3d4422017-08-03 08:27:51 -07003007TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003008 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003010 DataRate::BitsPerSec(kTargetBitrateBps),
3011 DataRate::BitsPerSec(kTargetBitrateBps),
3012 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
3013
3014 const int kWidth = 1280;
3015 const int kHeight = 720;
3016 int sequence = 1;
3017
3018 // Enable BALANCED preference, no initial limitation.
3019 test::FrameForwarder source;
3020 video_stream_encoder_->SetSource(&source,
3021 webrtc::DegradationPreference::BALANCED);
3022 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3023 WaitForEncodedFrame(sequence++);
3024 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3025 EXPECT_FALSE(stats.cpu_limited_resolution);
3026 EXPECT_FALSE(stats.cpu_limited_framerate);
3027 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3028
3029 // Trigger CPU overuse, should now adapt down.
3030 video_stream_encoder_->TriggerCpuOveruse();
3031 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3032 WaitForEncodedFrame(sequence++);
3033 stats = stats_proxy_->GetStats();
3034 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3035
3036 // Set new degradation preference should clear restrictions since we changed
3037 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003038 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003039 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3040 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3041 WaitForEncodedFrame(sequence++);
3042 stats = stats_proxy_->GetStats();
3043 EXPECT_FALSE(stats.cpu_limited_resolution);
3044 EXPECT_FALSE(stats.cpu_limited_framerate);
3045 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3046
3047 // Force an input frame rate to be available, or the adaptation call won't
3048 // know what framerate to adapt from.
3049 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3050 mock_stats.input_frame_rate = 30;
3051 stats_proxy_->SetMockStats(mock_stats);
3052 video_stream_encoder_->TriggerCpuOveruse();
3053 stats_proxy_->ResetMockStats();
3054 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3055 WaitForEncodedFrame(sequence++);
3056
3057 // We have now adapted once.
3058 stats = stats_proxy_->GetStats();
3059 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3060
3061 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003062 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3063 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003064 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3065 WaitForEncodedFrame(sequence++);
3066 stats = stats_proxy_->GetStats();
3067 EXPECT_FALSE(stats.cpu_limited_resolution);
3068 EXPECT_FALSE(stats.cpu_limited_framerate);
3069 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3070
3071 video_stream_encoder_->Stop();
3072}
3073
3074TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003075 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003077 DataRate::BitsPerSec(kTargetBitrateBps),
3078 DataRate::BitsPerSec(kTargetBitrateBps),
3079 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003080
asapersson0944a802017-04-07 00:57:58 -07003081 const int kWidth = 1280;
3082 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003083 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003084
asaperssonfab67072017-04-04 05:51:49 -07003085 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003086 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003087 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003088 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003089 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003090 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3091
asapersson02465b82017-04-10 01:12:52 -07003092 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003093 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003094 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003095 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003096 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003097 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003098 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003099 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3100
3101 // Set new source with adaptation still enabled.
3102 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003103 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003104 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003105
3106 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003107 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003108 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003109 stats = stats_proxy_->GetStats();
3110 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003111 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003112 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3113
sprangc5d62e22017-04-02 23:53:04 -07003114 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003115 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003116 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003117 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003118 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003119 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003120 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003121 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003122 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003123 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003124 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3125
sprangc5d62e22017-04-02 23:53:04 -07003126 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003127 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003128 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3129 mock_stats.input_frame_rate = 30;
3130 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003131 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003132 stats_proxy_->ResetMockStats();
3133
3134 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003135 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003136 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003137
3138 // Framerate now adapted.
3139 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003140 EXPECT_FALSE(stats.cpu_limited_resolution);
3141 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003142 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3143
3144 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003145 video_stream_encoder_->SetSource(&new_video_source,
3146 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003147 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003148 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003149 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003150
3151 stats = stats_proxy_->GetStats();
3152 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003154 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3155
3156 // Try to trigger overuse. Should not succeed.
3157 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003158 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003159 stats_proxy_->ResetMockStats();
3160
3161 stats = stats_proxy_->GetStats();
3162 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003163 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003164 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3165
3166 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003168 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003169 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003170 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003171 stats = stats_proxy_->GetStats();
3172 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003173 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003174 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003175
3176 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003177 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003178 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003179 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003180 stats = stats_proxy_->GetStats();
3181 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003182 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003183 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3184
3185 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003186 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003187 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003188 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003189 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003190 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003191 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003192 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003193 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003194 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003195 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3196
3197 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003198 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003199 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003200 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003201 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003202 stats = stats_proxy_->GetStats();
3203 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003204 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003205 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003206 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003207
mflodmancc3d4422017-08-03 08:27:51 -07003208 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003209}
3210
mflodmancc3d4422017-08-03 08:27:51 -07003211TEST_F(VideoStreamEncoderTest,
3212 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003213 const int kWidth = 1280;
3214 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003215 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003216 DataRate::BitsPerSec(kTargetBitrateBps),
3217 DataRate::BitsPerSec(kTargetBitrateBps),
3218 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003219
asaperssonfab67072017-04-04 05:51:49 -07003220 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003221 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003222
asaperssonfab67072017-04-04 05:51:49 -07003223 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003224 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003225
asaperssonfab67072017-04-04 05:51:49 -07003226 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003228
asaperssonfab67072017-04-04 05:51:49 -07003229 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003230 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003231
kthelgason876222f2016-11-29 01:44:11 -08003232 // Expect a scale down.
3233 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003234 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003235
asapersson02465b82017-04-10 01:12:52 -07003236 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003237 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003238 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003239 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003240
asaperssonfab67072017-04-04 05:51:49 -07003241 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003242 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003243 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003244 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003245
asaperssonfab67072017-04-04 05:51:49 -07003246 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003247 EXPECT_EQ(std::numeric_limits<int>::max(),
3248 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003249
asaperssonfab67072017-04-04 05:51:49 -07003250 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003251 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003252 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003253 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003254
asapersson02465b82017-04-10 01:12:52 -07003255 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003256 EXPECT_EQ(std::numeric_limits<int>::max(),
3257 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003258
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003260}
3261
mflodmancc3d4422017-08-03 08:27:51 -07003262TEST_F(VideoStreamEncoderTest,
3263 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003264 const int kWidth = 1280;
3265 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003266 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003267 DataRate::BitsPerSec(kTargetBitrateBps),
3268 DataRate::BitsPerSec(kTargetBitrateBps),
3269 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003270
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003271 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003272 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003273 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003274 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003275
3276 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003277 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003278 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003279 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3280 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3281
3282 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003283 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003284 EXPECT_THAT(source.sink_wants(),
3285 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003286 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3287 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3288 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3289
3290 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003291 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003292 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3293 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3294 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3295
mflodmancc3d4422017-08-03 08:27:51 -07003296 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003297}
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003300 const int kWidth = 1280;
3301 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003303 DataRate::BitsPerSec(kTargetBitrateBps),
3304 DataRate::BitsPerSec(kTargetBitrateBps),
3305 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003306
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003307 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003308 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003309 video_stream_encoder_->SetSource(&source,
3310 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003311 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3312 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003313 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003314
3315 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003316 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(source.sink_wants(),
3318 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003319 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3320 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3321 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3322
3323 // Trigger adapt down for same input resolution, expect no change.
3324 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3325 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003326 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003327 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3328 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3329 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3330
3331 // Trigger adapt down for larger input resolution, expect no change.
3332 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3333 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003335 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3336 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3337 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3338
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003340}
3341
mflodmancc3d4422017-08-03 08:27:51 -07003342TEST_F(VideoStreamEncoderTest,
3343 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003344 const int kWidth = 1280;
3345 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003347 DataRate::BitsPerSec(kTargetBitrateBps),
3348 DataRate::BitsPerSec(kTargetBitrateBps),
3349 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003350
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003351 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003352 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003353 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003354 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003355
3356 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003357 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003358 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003359 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3360 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3361
3362 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003363 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003364 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003365 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3366 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3367
mflodmancc3d4422017-08-03 08:27:51 -07003368 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003369}
3370
mflodmancc3d4422017-08-03 08:27:51 -07003371TEST_F(VideoStreamEncoderTest,
3372 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003373 const int kWidth = 1280;
3374 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003376 DataRate::BitsPerSec(kTargetBitrateBps),
3377 DataRate::BitsPerSec(kTargetBitrateBps),
3378 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003379
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003380 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003381 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003383 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003384
3385 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003386 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003387 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003388 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003389 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3390
3391 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003392 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003393 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003394 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003395 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3396
mflodmancc3d4422017-08-03 08:27:51 -07003397 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003398}
3399
mflodmancc3d4422017-08-03 08:27:51 -07003400TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003401 const int kWidth = 1280;
3402 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003404 DataRate::BitsPerSec(kTargetBitrateBps),
3405 DataRate::BitsPerSec(kTargetBitrateBps),
3406 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003407
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003408 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003409 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003410 video_stream_encoder_->SetSource(&source,
3411 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003412
3413 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3414 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003415 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003416 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3417 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3418 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419
3420 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003421 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003422 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3424 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
mflodmancc3d4422017-08-03 08:27:51 -07003427 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003428}
3429
mflodmancc3d4422017-08-03 08:27:51 -07003430TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003431 const int kWidth = 1280;
3432 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003434 DataRate::BitsPerSec(kTargetBitrateBps),
3435 DataRate::BitsPerSec(kTargetBitrateBps),
3436 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003437
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003438 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003439 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003440 video_stream_encoder_->SetSource(&source,
3441 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003442
3443 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3444 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003445 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3447 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003451 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3454 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3455 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
mflodmancc3d4422017-08-03 08:27:51 -07003457 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003458}
3459
mflodmancc3d4422017-08-03 08:27:51 -07003460TEST_F(VideoStreamEncoderTest,
3461 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003462 const int kWidth = 1280;
3463 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003465 DataRate::BitsPerSec(kTargetBitrateBps),
3466 DataRate::BitsPerSec(kTargetBitrateBps),
3467 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003468
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003469 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003470 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003471 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003472 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003473 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003474
3475 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003476 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003477 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3479 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003482 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003483 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003484 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003485 EXPECT_THAT(source.sink_wants(),
3486 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3488 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3489
3490 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003491 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003492 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3494 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3495 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3496
mflodmancc3d4422017-08-03 08:27:51 -07003497 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003498}
3499
mflodmancc3d4422017-08-03 08:27:51 -07003500TEST_F(VideoStreamEncoderTest,
3501 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003502 const int kWidth = 1280;
3503 const int kHeight = 720;
3504 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003506 DataRate::BitsPerSec(kTargetBitrateBps),
3507 DataRate::BitsPerSec(kTargetBitrateBps),
3508 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003509
3510 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3511 stats.input_frame_rate = kInputFps;
3512 stats_proxy_->SetMockStats(stats);
3513
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003514 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003515 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3516 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003517 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003518
3519 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003520 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003521 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3522 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003523 EXPECT_THAT(video_source_.sink_wants(),
3524 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003525
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003526 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003527 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003528 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003529 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003530 // Give the encoder queue time to process the change in degradation preference
3531 // by waiting for an encoded frame.
3532 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3533 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003534 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003535
3536 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003537 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003538 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3539 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003540 EXPECT_THAT(new_video_source.sink_wants(),
3541 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003542
3543 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003544 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003545 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003546
mflodmancc3d4422017-08-03 08:27:51 -07003547 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003548}
3549
mflodmancc3d4422017-08-03 08:27:51 -07003550TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003551 const int kWidth = 1280;
3552 const int kHeight = 720;
3553 const size_t kNumFrames = 10;
3554
Henrik Boström381d1092020-05-12 18:49:07 +02003555 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003556 DataRate::BitsPerSec(kTargetBitrateBps),
3557 DataRate::BitsPerSec(kTargetBitrateBps),
3558 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003559
asaperssond0de2952017-04-21 01:47:31 -07003560 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003561 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003562 video_source_.set_adaptation_enabled(true);
3563
3564 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3565 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566
3567 int downscales = 0;
3568 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003569 video_source_.IncomingCapturedFrame(
3570 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3571 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003572
asaperssonfab67072017-04-04 05:51:49 -07003573 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003574 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003575 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003576 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003577
3578 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3579 ++downscales;
3580
3581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3582 EXPECT_EQ(downscales,
3583 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3584 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003585 }
mflodmancc3d4422017-08-03 08:27:51 -07003586 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003587}
3588
mflodmancc3d4422017-08-03 08:27:51 -07003589TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003590 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3591 const int kWidth = 1280;
3592 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003594 DataRate::BitsPerSec(kTargetBitrateBps),
3595 DataRate::BitsPerSec(kTargetBitrateBps),
3596 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003597
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003598 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003599 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003600 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003601 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003602 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003603
Åsa Persson8c1bf952018-09-13 10:42:19 +02003604 int64_t timestamp_ms = kFrameIntervalMs;
3605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003606 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003607 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3609 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3610
3611 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003612 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003613 timestamp_ms += kFrameIntervalMs;
3614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003616 EXPECT_THAT(source.sink_wants(),
3617 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003618 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3619 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620
3621 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003622 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003623 timestamp_ms += kFrameIntervalMs;
3624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003625 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003626 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3628 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629
3630 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003632 timestamp_ms += kFrameIntervalMs;
3633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003635 EXPECT_THAT(source.sink_wants(),
3636 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003637 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3638 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3639
3640 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003641 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003642 timestamp_ms += kFrameIntervalMs;
3643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003644 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003645 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003646 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3647 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3648
mflodmancc3d4422017-08-03 08:27:51 -07003649 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003650}
3651
mflodmancc3d4422017-08-03 08:27:51 -07003652TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003653 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3654 const int kWidth = 1280;
3655 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003657 DataRate::BitsPerSec(kTargetBitrateBps),
3658 DataRate::BitsPerSec(kTargetBitrateBps),
3659 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003660
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003661 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003662 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003663 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003664 video_stream_encoder_->SetSource(&source,
3665 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003666
Åsa Persson8c1bf952018-09-13 10:42:19 +02003667 int64_t timestamp_ms = kFrameIntervalMs;
3668 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003669 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003670 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3672 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3673
3674 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003675 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003676 timestamp_ms += kFrameIntervalMs;
3677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3678 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003679 EXPECT_THAT(source.sink_wants(),
3680 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003681 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3682 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3683
3684 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003685 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003686 timestamp_ms += kFrameIntervalMs;
3687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003688 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003689 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3691 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3692
3693 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003694 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003695 timestamp_ms += kFrameIntervalMs;
3696 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3697 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003698 EXPECT_THAT(source.sink_wants(),
3699 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003700 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3701 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3702
3703 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003704 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003705 timestamp_ms += kFrameIntervalMs;
3706 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003707 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003708 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3710 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3711
mflodmancc3d4422017-08-03 08:27:51 -07003712 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003713}
3714
Sergey Silkin41c650b2019-10-14 13:12:19 +02003715TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3716 fake_encoder_.SetResolutionBitrateLimits(
3717 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3718
Henrik Boström381d1092020-05-12 18:49:07 +02003719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003720 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3721 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3722 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3723 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003724
3725 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003726 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003727 source.set_adaptation_enabled(true);
3728 video_stream_encoder_->SetSource(
3729 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3730
3731 // Insert 720p frame.
3732 int64_t timestamp_ms = kFrameIntervalMs;
3733 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3734 WaitForEncodedFrame(1280, 720);
3735
3736 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003738 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3739 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3740 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3741 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003742 video_stream_encoder_->TriggerQualityLow();
3743
3744 // Insert 720p frame. It should be downscaled and encoded.
3745 timestamp_ms += kFrameIntervalMs;
3746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3747 WaitForEncodedFrame(960, 540);
3748
3749 // Trigger adapt up. Higher resolution should not be requested duo to lack
3750 // of bitrate.
3751 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003752 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003753
3754 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003756 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3757 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3758 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3759 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003760
3761 // Trigger adapt up. Higher resolution should be requested.
3762 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003763 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003764
3765 video_stream_encoder_->Stop();
3766}
3767
3768TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3769 fake_encoder_.SetResolutionBitrateLimits(
3770 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3771
3772 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003773 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003774 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3775 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3776 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3777 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003778
3779 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003780 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003781 source.set_adaptation_enabled(true);
3782 video_stream_encoder_->SetSource(
3783 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3784
3785 // Insert 720p frame. It should be dropped and lower resolution should be
3786 // requested.
3787 int64_t timestamp_ms = kFrameIntervalMs;
3788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3789 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02003790 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003791
3792 // Insert 720p frame. It should be downscaled and encoded.
3793 timestamp_ms += kFrameIntervalMs;
3794 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3795 WaitForEncodedFrame(960, 540);
3796
3797 video_stream_encoder_->Stop();
3798}
3799
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003800class BalancedDegradationTest : public VideoStreamEncoderTest {
3801 protected:
3802 void SetupTest() {
3803 // Reset encoder for field trials to take effect.
3804 ConfigureEncoder(video_encoder_config_.Copy());
Åsa Perssonccfb3402019-09-25 15:13:04 +02003805 OnBitrateUpdated(kTargetBitrateBps);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003806
3807 // Enable BALANCED preference.
3808 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02003809 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
3810 }
3811
3812 void OnBitrateUpdated(int bitrate_bps) {
Henrik Boström381d1092020-05-12 18:49:07 +02003813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003814 DataRate::BitsPerSec(bitrate_bps), DataRate::BitsPerSec(bitrate_bps),
3815 DataRate::BitsPerSec(bitrate_bps), 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003816 }
3817
Åsa Persson45b176f2019-09-30 11:19:05 +02003818 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003819 timestamp_ms_ += kFrameIntervalMs;
3820 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02003821 }
3822
3823 void InsertFrameAndWaitForEncoded() {
3824 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003825 sink_.WaitForEncodedFrame(timestamp_ms_);
3826 }
3827
3828 const int kWidth = 640; // pixels:640x360=230400
3829 const int kHeight = 360;
3830 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
3831 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003832 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003833};
3834
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003835TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003836 test::ScopedFieldTrials field_trials(
3837 "WebRTC-Video-BalancedDegradationSettings/"
3838 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3839 SetupTest();
3840
3841 // Force input frame rate.
3842 const int kInputFps = 24;
3843 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3844 stats.input_frame_rate = kInputFps;
3845 stats_proxy_->SetMockStats(stats);
3846
Åsa Persson45b176f2019-09-30 11:19:05 +02003847 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003848 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003849
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003850 // Trigger adapt down, expect scaled down framerate and resolution,
3851 // since Fps diff (input-requested:0) < threshold.
3852 video_stream_encoder_->TriggerQualityLow();
3853 EXPECT_THAT(source_.sink_wants(),
3854 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003855
3856 video_stream_encoder_->Stop();
3857}
3858
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003859TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003860 test::ScopedFieldTrials field_trials(
3861 "WebRTC-Video-BalancedDegradationSettings/"
3862 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
3863 SetupTest();
3864
3865 // Force input frame rate.
3866 const int kInputFps = 25;
3867 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3868 stats.input_frame_rate = kInputFps;
3869 stats_proxy_->SetMockStats(stats);
3870
Åsa Persson45b176f2019-09-30 11:19:05 +02003871 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003872 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003873
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02003874 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
3875 // Fps diff (input-requested:1) == threshold.
3876 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003877 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003878
3879 video_stream_encoder_->Stop();
3880}
3881
3882TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
3883 test::ScopedFieldTrials field_trials(
3884 "WebRTC-Video-BalancedDegradationSettings/"
3885 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
3886 SetupTest();
3887
3888 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
3889
Åsa Persson45b176f2019-09-30 11:19:05 +02003890 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003891 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003892
3893 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
3894 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003895 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003896
3897 video_stream_encoder_->Stop();
3898}
3899
Åsa Perssonccfb3402019-09-25 15:13:04 +02003900TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003901 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02003902 "WebRTC-Video-BalancedDegradationSettings/"
3903 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003904 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02003905
Åsa Persson1b247f12019-08-14 17:26:39 +02003906 const int kMinBitrateBps = 425000;
3907 const int kTooLowMinBitrateBps = 424000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003908 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003909
Åsa Persson45b176f2019-09-30 11:19:05 +02003910 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003911 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02003912 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913
3914 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3915 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003916 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003917 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02003918 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3919
3920 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
3921 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003922 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003923 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02003924 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3925
Åsa Persson30ab0152019-08-27 12:22:33 +02003926 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
3927 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003928 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02003929 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02003930 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02003931 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3932
3933 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02003934 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003935 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02003936 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02003937
Åsa Persson30ab0152019-08-27 12:22:33 +02003938 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02003939 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson1b247f12019-08-14 17:26:39 +02003940 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02003941 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02003942 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02003943 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3944
3945 video_stream_encoder_->Stop();
3946}
3947
Åsa Perssonccfb3402019-09-25 15:13:04 +02003948TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02003949 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
3950 test::ScopedFieldTrials field_trials(
3951 "WebRTC-Video-BalancedDegradationSettings/"
3952 "pixels:57600|129600|230400,fps:7|24|24/");
3953 SetupTest();
3954 OnBitrateUpdated(kLowTargetBitrateBps);
3955
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003956 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02003957
3958 // Insert frame, expect scaled down:
3959 // framerate (640x360@24fps) -> resolution (480x270@24fps).
3960 InsertFrame();
3961 EXPECT_FALSE(WaitForFrame(1000));
3962 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
3963 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3964
3965 // Insert frame, expect scaled down:
3966 // resolution (320x180@24fps).
3967 InsertFrame();
3968 EXPECT_FALSE(WaitForFrame(1000));
3969 EXPECT_LT(source_.sink_wants().max_pixel_count,
3970 source_.last_wants().max_pixel_count);
3971 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
3972
3973 // Frame should not be dropped (min pixels per frame reached).
3974 InsertFrameAndWaitForEncoded();
3975
3976 video_stream_encoder_->Stop();
3977}
3978
3979TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02003980 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02003981 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02003982 "WebRTC-Video-BalancedDegradationSettings/"
3983 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02003984 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02003985
Åsa Persson30ab0152019-08-27 12:22:33 +02003986 const int kResolutionMinBitrateBps = 435000;
3987 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02003988 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02003989
Åsa Persson45b176f2019-09-30 11:19:05 +02003990 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003991 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02003992 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3993
3994 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
3995 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02003996 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003997 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02003998 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3999
4000 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4001 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004002 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004003 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004004 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4005
4006 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4007 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004008 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004009 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004010 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4011
Åsa Persson30ab0152019-08-27 12:22:33 +02004012 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4013 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004014 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004015 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004016 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4017
4018 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4019 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004020 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004021 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
4023 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004024 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004025 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004026 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004027 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004028 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4029
4030 video_stream_encoder_->Stop();
4031}
4032
Åsa Perssonccfb3402019-09-25 15:13:04 +02004033TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004034 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004035 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004036 "WebRTC-Video-BalancedDegradationSettings/"
4037 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004038 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004039
Åsa Persson30ab0152019-08-27 12:22:33 +02004040 const int kMinBitrateBps = 425000;
4041 const int kTooLowMinBitrateBps = 424000;
4042 const int kResolutionMinBitrateBps = 435000;
4043 const int kTooLowMinResolutionBitrateBps = 434000;
Åsa Perssonccfb3402019-09-25 15:13:04 +02004044 OnBitrateUpdated(kTooLowMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004045
Åsa Persson45b176f2019-09-30 11:19:05 +02004046 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004047 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004048 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4049
4050 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4051 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004052 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004053 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004054 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4055
4056 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4057 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004058 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004059 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004060 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4061
4062 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4063 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004064 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004065 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004066 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4067
4068 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4069 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004070 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004071 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4072
4073 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004074 OnBitrateUpdated(kMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004075 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004076 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004077 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004078 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4079
4080 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004081 OnBitrateUpdated(kTooLowMinResolutionBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004082 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004083 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004084 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4085
4086 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Åsa Perssonccfb3402019-09-25 15:13:04 +02004087 OnBitrateUpdated(kResolutionMinBitrateBps);
Åsa Persson30ab0152019-08-27 12:22:33 +02004088 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004089 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004090 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004091 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4092
Åsa Persson1b247f12019-08-14 17:26:39 +02004093 video_stream_encoder_->Stop();
4094}
4095
mflodmancc3d4422017-08-03 08:27:51 -07004096TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004097 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4098 const int kWidth = 1280;
4099 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004101 DataRate::BitsPerSec(kTargetBitrateBps),
4102 DataRate::BitsPerSec(kTargetBitrateBps),
4103 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004104
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004105 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004106 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004107 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004108 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004109 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004110
Åsa Persson8c1bf952018-09-13 10:42:19 +02004111 int64_t timestamp_ms = kFrameIntervalMs;
4112 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004113 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004114 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004115 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4116 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4117 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4118 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4119
4120 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004121 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004122 timestamp_ms += kFrameIntervalMs;
4123 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4124 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004125 EXPECT_THAT(source.sink_wants(),
4126 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004127 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4128 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4129 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4130 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4131
4132 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004133 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004134 timestamp_ms += kFrameIntervalMs;
4135 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4136 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004137 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004138 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4139 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4140 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4141 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4142
Jonathan Yubc771b72017-12-08 17:04:29 -08004143 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004144 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004145 timestamp_ms += kFrameIntervalMs;
4146 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4147 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004148 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004149 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4150 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004151 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004152 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4153
Jonathan Yubc771b72017-12-08 17:04:29 -08004154 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004155 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004156 timestamp_ms += kFrameIntervalMs;
4157 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4158 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004159 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004160 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004161 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4162 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4163 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4164 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4165
Jonathan Yubc771b72017-12-08 17:04:29 -08004166 // Trigger quality adapt down, expect no change (min resolution reached).
4167 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004168 timestamp_ms += kFrameIntervalMs;
4169 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4170 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004171 EXPECT_THAT(source.sink_wants(), FpsMax());
4172 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004173 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4174 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4175 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4176 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4177
Evan Shrubsole64469032020-06-11 10:45:29 +02004178 // Trigger quality adapt up, expect upscaled resolution (480x270).
4179 video_stream_encoder_->TriggerQualityHigh();
4180 timestamp_ms += kFrameIntervalMs;
4181 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4182 WaitForEncodedFrame(timestamp_ms);
4183 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4184 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4185 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4186 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4187 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4188
4189 // Trigger quality and cpu adapt up since both are most limited, expect
4190 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004191 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004192 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004193 timestamp_ms += kFrameIntervalMs;
4194 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4195 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004196 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004197 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4198 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4199 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004200 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004201
Evan Shrubsole64469032020-06-11 10:45:29 +02004202 // Trigger quality and cpu adapt up since both are most limited, expect
4203 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004204 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004205 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004206 timestamp_ms += kFrameIntervalMs;
4207 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4208 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004209 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004210 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004211 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004212 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004213 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4214 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004215
Evan Shrubsole64469032020-06-11 10:45:29 +02004216 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4217 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004218 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004219 timestamp_ms += kFrameIntervalMs;
4220 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4221 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004222 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004223 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4224 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004225 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004226 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004227
4228 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004229 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004230 timestamp_ms += kFrameIntervalMs;
4231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004232 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004233 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004234 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004235 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004237 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004238 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004239
mflodmancc3d4422017-08-03 08:27:51 -07004240 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004241}
4242
mflodmancc3d4422017-08-03 08:27:51 -07004243TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004244 const int kWidth = 640;
4245 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004246
Henrik Boström381d1092020-05-12 18:49:07 +02004247 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004248 DataRate::BitsPerSec(kTargetBitrateBps),
4249 DataRate::BitsPerSec(kTargetBitrateBps),
4250 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004251
perkj803d97f2016-11-01 11:45:46 -07004252 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004253 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004254 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004255 }
4256
mflodmancc3d4422017-08-03 08:27:51 -07004257 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004258 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004259 video_source_.IncomingCapturedFrame(CreateFrame(
4260 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004261 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004262 }
4263
mflodmancc3d4422017-08-03 08:27:51 -07004264 video_stream_encoder_->Stop();
4265 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004266 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004267
Ying Wangef3998f2019-12-09 13:06:53 +01004268 EXPECT_METRIC_EQ(
4269 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4270 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004271 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4272}
4273
mflodmancc3d4422017-08-03 08:27:51 -07004274TEST_F(VideoStreamEncoderTest,
4275 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004277 DataRate::BitsPerSec(kTargetBitrateBps),
4278 DataRate::BitsPerSec(kTargetBitrateBps),
4279 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004280 const int kWidth = 640;
4281 const int kHeight = 360;
4282
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004283 video_stream_encoder_->SetSource(&video_source_,
4284 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004285
4286 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4287 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004288 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004289 }
4290
mflodmancc3d4422017-08-03 08:27:51 -07004291 video_stream_encoder_->Stop();
4292 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004293 stats_proxy_.reset();
4294
4295 EXPECT_EQ(0,
4296 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4297}
4298
Per Kjellanderdcef6412020-10-07 15:09:05 +02004299TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4300 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004301 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004302 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004303
4304 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004305 const VideoBitrateAllocation expected_bitrate =
Mirta Dvornicic6799d732020-02-12 15:36:49 +01004306 SimulcastRateAllocator(fake_encoder_.codec_config())
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004307 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrateBps,
4308 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004309
Henrik Boström381d1092020-05-12 18:49:07 +02004310 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004311 DataRate::BitsPerSec(kLowTargetBitrateBps),
4312 DataRate::BitsPerSec(kLowTargetBitrateBps),
4313 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004314
sprang57c2fff2017-01-16 06:24:02 -08004315 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004316 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4317 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004318 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4319 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4320
Erik Språngd7329ca2019-02-21 21:19:53 +01004321 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004322 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004323 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004324
Per Kjellanderdcef6412020-10-07 15:09:05 +02004325 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004326 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004327 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4328 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004329 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004330 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004331
Per Kjellanderdcef6412020-10-07 15:09:05 +02004332 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004333 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004334 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004335 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004336 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4337 WaitForEncodedFrame(CurrentTimeMs());
4338 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004339 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004340 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004341
mflodmancc3d4422017-08-03 08:27:51 -07004342 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004343}
4344
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004345TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004346 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004347 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004348 kVideoLayersAllocation);
4349
4350 const int kDefaultFps = 30;
4351
4352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4353 DataRate::BitsPerSec(kLowTargetBitrateBps),
4354 DataRate::BitsPerSec(kLowTargetBitrateBps),
4355 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4356
4357 video_source_.IncomingCapturedFrame(
4358 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4359 WaitForEncodedFrame(CurrentTimeMs());
4360 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4361 VideoLayersAllocation last_layer_allocation =
4362 sink_.GetLastVideoLayersAllocation();
4363 // kLowTargetBitrateBps is only enough for one spatial layer.
4364 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4365
4366 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004367 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004368 // Check that encoder has been updated too, not just allocation observer.
4369 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrateBps);
4370 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4371
Erik Språng9d69cbe2020-10-22 17:44:42 +02004372 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004373 int number_of_layers_allocation = 1;
4374 const int64_t start_time_ms = CurrentTimeMs();
4375 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4376 video_source_.IncomingCapturedFrame(
4377 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4378 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004379 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4380 number_of_layers_allocation = sink_.number_of_layers_allocations();
4381 VideoLayersAllocation new_allocation =
4382 sink_.GetLastVideoLayersAllocation();
4383 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4384 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4385 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4386 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4387 .target_bitrate_per_temporal_layer,
4388 last_layer_allocation.active_spatial_layers[0]
4389 .target_bitrate_per_temporal_layer);
4390 last_layer_allocation = new_allocation;
4391 }
4392 }
4393 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4394 video_stream_encoder_->Stop();
4395}
4396
4397TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004398 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004399 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4400 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4401 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004402 VideoEncoderConfig video_encoder_config;
4403 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4404 /* num_streams*/ 3, &video_encoder_config);
4405 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4406 video_encoder_config.content_type =
4407 VideoEncoderConfig::ContentType::kRealtimeVideo;
4408 video_encoder_config.encoder_specific_settings =
4409 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4410 VideoEncoder::GetDefaultVp8Settings());
4411 for (auto& layer : video_encoder_config.simulcast_layers) {
4412 layer.num_temporal_layers = 2;
4413 }
4414 // Simulcast layers are used for enabling/disabling streams.
4415 video_encoder_config.simulcast_layers[0].active = true;
4416 video_encoder_config.simulcast_layers[1].active = false;
4417 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004418 ConfigureEncoder(std::move(video_encoder_config),
4419 VideoStreamEncoder::BitrateAllocationCallbackType::
4420 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004421
4422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4423 DataRate::BitsPerSec(kTargetBitrateBps),
4424 DataRate::BitsPerSec(kTargetBitrateBps),
4425 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4426
4427 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4428 WaitForEncodedFrame(CurrentTimeMs());
4429 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4430 VideoLayersAllocation last_layer_allocation =
4431 sink_.GetLastVideoLayersAllocation();
4432
4433 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4434 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4435 .target_bitrate_per_temporal_layer,
4436 SizeIs(2));
4437 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4438 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4439 video_stream_encoder_->Stop();
4440}
4441
4442TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004443 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004444 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4445 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4446 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004447 VideoEncoderConfig video_encoder_config;
4448 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4449 /* num_streams*/ 3, &video_encoder_config);
4450 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4451 video_encoder_config.content_type =
4452 VideoEncoderConfig::ContentType::kRealtimeVideo;
4453 video_encoder_config.encoder_specific_settings =
4454 new rtc::RefCountedObject<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
4455 VideoEncoder::GetDefaultVp8Settings());
4456 for (auto& layer : video_encoder_config.simulcast_layers) {
4457 layer.num_temporal_layers = 2;
4458 }
4459 // Simulcast layers are used for enabling/disabling streams.
4460 video_encoder_config.simulcast_layers[0].active = true;
4461 video_encoder_config.simulcast_layers[1].active = false;
4462 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004463 ConfigureEncoder(std::move(video_encoder_config),
4464 VideoStreamEncoder::BitrateAllocationCallbackType::
4465 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004466
4467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4468 DataRate::BitsPerSec(kTargetBitrateBps),
4469 DataRate::BitsPerSec(kTargetBitrateBps),
4470 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4471
4472 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4473 WaitForEncodedFrame(CurrentTimeMs());
4474 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4475 VideoLayersAllocation last_layer_allocation =
4476 sink_.GetLastVideoLayersAllocation();
4477
4478 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4479 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4480 .target_bitrate_per_temporal_layer,
4481 SizeIs(2));
4482 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4483
4484 video_stream_encoder_->Stop();
4485}
4486
4487TEST_F(VideoStreamEncoderTest,
4488 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4489 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4490 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004491 VideoEncoderConfig video_encoder_config;
4492 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4493 /* num_streams*/ 1, &video_encoder_config);
4494 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4495 video_encoder_config.content_type =
4496 VideoEncoderConfig::ContentType::kRealtimeVideo;
4497 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4498 vp9_settings.numberOfSpatialLayers = 2;
4499 vp9_settings.numberOfTemporalLayers = 2;
4500 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4501 vp9_settings.automaticResizeOn = false;
4502 video_encoder_config.encoder_specific_settings =
4503 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4504 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004505 ConfigureEncoder(std::move(video_encoder_config),
4506 VideoStreamEncoder::BitrateAllocationCallbackType::
4507 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004508
4509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4510 DataRate::BitsPerSec(kTargetBitrateBps),
4511 DataRate::BitsPerSec(kTargetBitrateBps),
4512 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4513
4514 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4515 WaitForEncodedFrame(CurrentTimeMs());
4516 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4517 VideoLayersAllocation last_layer_allocation =
4518 sink_.GetLastVideoLayersAllocation();
4519
4520 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4521 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4522 .target_bitrate_per_temporal_layer,
4523 SizeIs(2));
4524 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4525 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4526 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4527 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4528 .target_bitrate_per_temporal_layer,
4529 SizeIs(2));
4530 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4531 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4532 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4533
4534 // Since full SVC is used, expect the top layer to utilize the full target
4535 // rate.
4536 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4537 .target_bitrate_per_temporal_layer[1],
4538 DataRate::BitsPerSec(kTargetBitrateBps));
4539 video_stream_encoder_->Stop();
4540}
4541
4542TEST_F(VideoStreamEncoderTest,
4543 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4544 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4545 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004546 VideoEncoderConfig video_encoder_config;
4547 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4548 /* num_streams*/ 1, &video_encoder_config);
4549 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4550 video_encoder_config.content_type =
4551 VideoEncoderConfig::ContentType::kRealtimeVideo;
4552 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4553 vp9_settings.numberOfSpatialLayers = 2;
4554 vp9_settings.numberOfTemporalLayers = 2;
4555 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4556 vp9_settings.automaticResizeOn = false;
4557 video_encoder_config.encoder_specific_settings =
4558 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4559 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004560 ConfigureEncoder(std::move(video_encoder_config),
4561 VideoStreamEncoder::BitrateAllocationCallbackType::
4562 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004563
4564 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4565 DataRate::BitsPerSec(kTargetBitrateBps),
4566 DataRate::BitsPerSec(kTargetBitrateBps),
4567 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4568
4569 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4570 WaitForEncodedFrame(CurrentTimeMs());
4571 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4572 VideoLayersAllocation last_layer_allocation =
4573 sink_.GetLastVideoLayersAllocation();
4574
4575 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4576 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4577 .target_bitrate_per_temporal_layer,
4578 SizeIs(1));
4579 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4580 .target_bitrate_per_temporal_layer,
4581 SizeIs(1));
4582 // Since full SVC is used, expect the top layer to utilize the full target
4583 // rate.
4584 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4585 .target_bitrate_per_temporal_layer[0],
4586 DataRate::BitsPerSec(kTargetBitrateBps));
4587 video_stream_encoder_->Stop();
4588}
4589
4590TEST_F(VideoStreamEncoderTest,
4591 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4592 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4593 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004594 VideoEncoderConfig video_encoder_config;
4595 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4596 /* num_streams*/ 1, &video_encoder_config);
4597 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4598 video_encoder_config.content_type =
4599 VideoEncoderConfig::ContentType::kRealtimeVideo;
4600 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4601 vp9_settings.numberOfSpatialLayers = 2;
4602 vp9_settings.numberOfTemporalLayers = 2;
4603 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4604 vp9_settings.automaticResizeOn = false;
4605 video_encoder_config.encoder_specific_settings =
4606 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4607 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004608 ConfigureEncoder(std::move(video_encoder_config),
4609 VideoStreamEncoder::BitrateAllocationCallbackType::
4610 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004611
4612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4613 DataRate::BitsPerSec(kTargetBitrateBps),
4614 DataRate::BitsPerSec(kTargetBitrateBps),
4615 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4616
4617 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4618 WaitForEncodedFrame(CurrentTimeMs());
4619 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4620 VideoLayersAllocation last_layer_allocation =
4621 sink_.GetLastVideoLayersAllocation();
4622
4623 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4624 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4625 .target_bitrate_per_temporal_layer,
4626 SizeIs(2));
4627 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4628 .target_bitrate_per_temporal_layer,
4629 SizeIs(2));
4630 // Since KSVC is, spatial layers are independend except on key frames.
4631 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4632 .target_bitrate_per_temporal_layer[1],
4633 DataRate::BitsPerSec(kTargetBitrateBps));
4634 video_stream_encoder_->Stop();
4635}
4636
4637TEST_F(VideoStreamEncoderTest,
4638 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4639 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4640 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4641 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004642 VideoEncoderConfig video_encoder_config;
4643 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4644 /* num_streams*/ 1, &video_encoder_config);
4645 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4646 video_encoder_config.content_type =
4647 VideoEncoderConfig::ContentType::kRealtimeVideo;
4648 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4649 vp9_settings.numberOfSpatialLayers = 3;
4650 vp9_settings.numberOfTemporalLayers = 2;
4651 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4652 vp9_settings.automaticResizeOn = false;
4653 video_encoder_config.encoder_specific_settings =
4654 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4655 vp9_settings);
4656 // Simulcast layers are used for enabling/disabling streams.
4657 video_encoder_config.simulcast_layers.resize(3);
4658 video_encoder_config.simulcast_layers[0].active = false;
4659 video_encoder_config.simulcast_layers[1].active = true;
4660 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004661 ConfigureEncoder(std::move(video_encoder_config),
4662 VideoStreamEncoder::BitrateAllocationCallbackType::
4663 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004664
4665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4666 DataRate::BitsPerSec(kTargetBitrateBps),
4667 DataRate::BitsPerSec(kTargetBitrateBps),
4668 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4669
4670 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4671 WaitForEncodedFrame(CurrentTimeMs());
4672 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4673 VideoLayersAllocation last_layer_allocation =
4674 sink_.GetLastVideoLayersAllocation();
4675
4676 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4677 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4678 .target_bitrate_per_temporal_layer,
4679 SizeIs(2));
4680 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4681 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4682
4683 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4684 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4685 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4686 .target_bitrate_per_temporal_layer,
4687 SizeIs(2));
4688 // Since full SVC is used, expect the top layer to utilize the full target
4689 // rate.
4690 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4691 .target_bitrate_per_temporal_layer[1],
4692 DataRate::BitsPerSec(kTargetBitrateBps));
4693 video_stream_encoder_->Stop();
4694}
4695
4696TEST_F(VideoStreamEncoderTest,
4697 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4698 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4699 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4700 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004701 VideoEncoderConfig video_encoder_config;
4702 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4703 /* num_streams*/ 1, &video_encoder_config);
4704 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4705 video_encoder_config.content_type =
4706 VideoEncoderConfig::ContentType::kRealtimeVideo;
4707 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4708 vp9_settings.numberOfSpatialLayers = 3;
4709 vp9_settings.numberOfTemporalLayers = 2;
4710 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4711 vp9_settings.automaticResizeOn = false;
4712 video_encoder_config.encoder_specific_settings =
4713 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4714 vp9_settings);
4715 // Simulcast layers are used for enabling/disabling streams.
4716 video_encoder_config.simulcast_layers.resize(3);
4717 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004718 ConfigureEncoder(std::move(video_encoder_config),
4719 VideoStreamEncoder::BitrateAllocationCallbackType::
4720 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721
4722 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4723 DataRate::BitsPerSec(kTargetBitrateBps),
4724 DataRate::BitsPerSec(kTargetBitrateBps),
4725 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4726
4727 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4728 WaitForEncodedFrame(CurrentTimeMs());
4729 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4730 VideoLayersAllocation last_layer_allocation =
4731 sink_.GetLastVideoLayersAllocation();
4732
4733 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4734 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4735 .target_bitrate_per_temporal_layer,
4736 SizeIs(2));
4737 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4738 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4739
4740 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4741 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4742 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4743 .target_bitrate_per_temporal_layer,
4744 SizeIs(2));
4745 video_stream_encoder_->Stop();
4746}
4747
4748TEST_F(VideoStreamEncoderTest,
4749 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4750 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4751 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4752 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004753 VideoEncoderConfig video_encoder_config;
4754 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4755 /* num_streams*/ 1, &video_encoder_config);
4756 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrateBps;
4757 video_encoder_config.content_type =
4758 VideoEncoderConfig::ContentType::kRealtimeVideo;
4759 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4760 vp9_settings.numberOfSpatialLayers = 3;
4761 vp9_settings.numberOfTemporalLayers = 2;
4762 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4763 vp9_settings.automaticResizeOn = false;
4764 video_encoder_config.encoder_specific_settings =
4765 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
4766 vp9_settings);
4767 // Simulcast layers are used for enabling/disabling streams.
4768 video_encoder_config.simulcast_layers.resize(3);
4769 video_encoder_config.simulcast_layers[0].active = false;
4770 video_encoder_config.simulcast_layers[1].active = false;
4771 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004772 ConfigureEncoder(std::move(video_encoder_config),
4773 VideoStreamEncoder::BitrateAllocationCallbackType::
4774 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775
4776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4777 DataRate::BitsPerSec(kTargetBitrateBps),
4778 DataRate::BitsPerSec(kTargetBitrateBps),
4779 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4780
4781 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4782 WaitForEncodedFrame(CurrentTimeMs());
4783 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4784 VideoLayersAllocation last_layer_allocation =
4785 sink_.GetLastVideoLayersAllocation();
4786
4787 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4788 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4789 .target_bitrate_per_temporal_layer,
4790 SizeIs(2));
4791 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4792 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4793 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4794 .target_bitrate_per_temporal_layer[1],
4795 DataRate::BitsPerSec(kTargetBitrateBps));
4796 video_stream_encoder_->Stop();
4797}
4798
4799TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4800 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004801 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004802 kVideoLayersAllocation);
4803 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4804 DataRate::BitsPerSec(kTargetBitrateBps),
4805 DataRate::BitsPerSec(kTargetBitrateBps),
4806 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
4807
4808 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4809 WaitForEncodedFrame(CurrentTimeMs());
4810 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4811 VideoLayersAllocation last_layer_allocation =
4812 sink_.GetLastVideoLayersAllocation();
4813
4814 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4815 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
4816 .target_bitrate_per_temporal_layer,
4817 SizeIs(1));
4818 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4819 .target_bitrate_per_temporal_layer[0],
4820 DataRate::BitsPerSec(kTargetBitrateBps));
4821 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4822 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
4823 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4824 video_stream_encoder_->Stop();
4825}
4826
4827TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02004828 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
4829 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004830 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004831 kVideoLayersAllocation);
4832
4833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4834 DataRate::BitsPerSec(kLowTargetBitrateBps),
4835 DataRate::BitsPerSec(kLowTargetBitrateBps),
4836 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
4837
4838 video_source_.IncomingCapturedFrame(
4839 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4840 WaitForEncodedFrame(CurrentTimeMs());
4841 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4842 VideoLayersAllocation last_layer_allocation =
4843 sink_.GetLastVideoLayersAllocation();
4844 // kLowTargetBitrateBps is only enough for one spatial layer.
4845 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4846 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4847 .target_bitrate_per_temporal_layer[0],
4848 DataRate::BitsPerSec(kLowTargetBitrateBps));
4849
4850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4851 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4852 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4853 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4854 video_source_.IncomingCapturedFrame(
4855 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4856 WaitForEncodedFrame(CurrentTimeMs());
4857
4858 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4859 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
4860 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
4861 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
4862 .target_bitrate_per_temporal_layer[0],
4863 DataRate::Zero());
4864
4865 video_stream_encoder_->Stop();
4866}
4867
Per Kjellander4190ce92020-12-15 17:24:55 +01004868TEST_F(VideoStreamEncoderTest,
4869 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
4870 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004871 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01004872 kVideoLayersAllocation);
4873
4874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
4875 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4876 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
4877 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
4878
4879 video_source_.IncomingCapturedFrame(
4880 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4881 WaitForEncodedFrame(CurrentTimeMs());
4882 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4883 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4884 SizeIs(2));
4885 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4886 codec_width_);
4887 EXPECT_EQ(
4888 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4889 codec_height_);
4890
4891 video_source_.IncomingCapturedFrame(
4892 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
4893 WaitForEncodedFrame(CurrentTimeMs());
4894 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
4895 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
4896 SizeIs(2));
4897 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
4898 codec_width_ / 2);
4899 EXPECT_EQ(
4900 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
4901 codec_height_ / 2);
4902
4903 video_stream_encoder_->Stop();
4904}
4905
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004906TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
4907 // 2 TLs configured, temporal layers supported by encoder.
4908 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02004909 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004910 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004911 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004912 fake_encoder_.SetTemporalLayersSupported(0, true);
4913
4914 // Bitrate allocated across temporal layers.
4915 const int kTl0Bps = kTargetBitrateBps *
4916 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004917 kNumTemporalLayers, /*temporal_id*/ 0,
4918 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004919 const int kTl1Bps = kTargetBitrateBps *
4920 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004921 kNumTemporalLayers, /*temporal_id*/ 1,
4922 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004923 VideoBitrateAllocation expected_bitrate;
4924 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
4925 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
4926
4927 VerifyAllocatedBitrate(expected_bitrate);
4928 video_stream_encoder_->Stop();
4929}
4930
4931TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
4932 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004933 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004934 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004935 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004936 fake_encoder_.SetTemporalLayersSupported(0, false);
4937
4938 // Temporal layers not supported by the encoder.
4939 // Total bitrate should be at ti:0.
4940 VideoBitrateAllocation expected_bitrate;
4941 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrateBps);
4942
4943 VerifyAllocatedBitrate(expected_bitrate);
4944 video_stream_encoder_->Stop();
4945}
4946
4947TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02004948 webrtc::test::ScopedFieldTrials field_trials(
4949 "WebRTC-Video-QualityScalerSettings/"
4950 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
4951 // Reset encoder for field trials to take effect.
4952 ConfigureEncoder(video_encoder_config_.Copy());
4953
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004954 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02004955 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004956 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004957 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004958 fake_encoder_.SetTemporalLayersSupported(0, true);
4959 fake_encoder_.SetTemporalLayersSupported(1, false);
4960
4961 const int kS0Bps = 150000;
4962 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004963 kS0Bps *
4964 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4965 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004966 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01004967 kS0Bps *
4968 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
4969 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01004970 const int kS1Bps = kTargetBitrateBps - kS0Tl1Bps;
4971 // Temporal layers not supported by si:1.
4972 VideoBitrateAllocation expected_bitrate;
4973 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
4974 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
4975 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
4976
4977 VerifyAllocatedBitrate(expected_bitrate);
4978 video_stream_encoder_->Stop();
4979}
4980
Niels Möller7dc26b72017-12-06 10:27:48 +01004981TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
4982 const int kFrameWidth = 1280;
4983 const int kFrameHeight = 720;
4984 const int kFramerate = 24;
4985
Henrik Boström381d1092020-05-12 18:49:07 +02004986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004987 DataRate::BitsPerSec(kTargetBitrateBps),
4988 DataRate::BitsPerSec(kTargetBitrateBps),
4989 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01004990 test::FrameForwarder source;
4991 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004992 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01004993
4994 // Insert a single frame, triggering initial configuration.
4995 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
4996 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
4997
4998 EXPECT_EQ(
4999 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5000 kDefaultFramerate);
5001
5002 // Trigger reconfigure encoder (without resetting the entire instance).
5003 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005004 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5005 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005006 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005007 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005008 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005009 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5010
5011 // Detector should be updated with fps limit from codec config.
5012 EXPECT_EQ(
5013 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5014 kFramerate);
5015
5016 // Trigger overuse, max framerate should be reduced.
5017 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5018 stats.input_frame_rate = kFramerate;
5019 stats_proxy_->SetMockStats(stats);
5020 video_stream_encoder_->TriggerCpuOveruse();
5021 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5022 int adapted_framerate =
5023 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5024 EXPECT_LT(adapted_framerate, kFramerate);
5025
5026 // Trigger underuse, max framerate should go back to codec configured fps.
5027 // Set extra low fps, to make sure it's actually reset, not just incremented.
5028 stats = stats_proxy_->GetStats();
5029 stats.input_frame_rate = adapted_framerate / 2;
5030 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005031 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005032 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5033 EXPECT_EQ(
5034 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5035 kFramerate);
5036
5037 video_stream_encoder_->Stop();
5038}
5039
5040TEST_F(VideoStreamEncoderTest,
5041 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5042 const int kFrameWidth = 1280;
5043 const int kFrameHeight = 720;
5044 const int kLowFramerate = 15;
5045 const int kHighFramerate = 25;
5046
Henrik Boström381d1092020-05-12 18:49:07 +02005047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005048 DataRate::BitsPerSec(kTargetBitrateBps),
5049 DataRate::BitsPerSec(kTargetBitrateBps),
5050 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005051 test::FrameForwarder source;
5052 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005053 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005054
5055 // Trigger initial configuration.
5056 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005057 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5058 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005059 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
Niels Möller7dc26b72017-12-06 10:27:48 +01005060 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005061 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005062 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005063 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5064
5065 EXPECT_EQ(
5066 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5067 kLowFramerate);
5068
5069 // Trigger overuse, max framerate should be reduced.
5070 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5071 stats.input_frame_rate = kLowFramerate;
5072 stats_proxy_->SetMockStats(stats);
5073 video_stream_encoder_->TriggerCpuOveruse();
5074 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5075 int adapted_framerate =
5076 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5077 EXPECT_LT(adapted_framerate, kLowFramerate);
5078
5079 // Reconfigure the encoder with a new (higher max framerate), max fps should
5080 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005081 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005082 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5083 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005084 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005085 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5086
5087 EXPECT_EQ(
5088 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5089 adapted_framerate);
5090
5091 // Trigger underuse, max framerate should go back to codec configured fps.
5092 stats = stats_proxy_->GetStats();
5093 stats.input_frame_rate = adapted_framerate;
5094 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005095 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005096 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5097 EXPECT_EQ(
5098 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5099 kHighFramerate);
5100
5101 video_stream_encoder_->Stop();
5102}
5103
mflodmancc3d4422017-08-03 08:27:51 -07005104TEST_F(VideoStreamEncoderTest,
5105 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005106 const int kFrameWidth = 1280;
5107 const int kFrameHeight = 720;
5108 const int kFramerate = 24;
5109
Henrik Boström381d1092020-05-12 18:49:07 +02005110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005111 DataRate::BitsPerSec(kTargetBitrateBps),
5112 DataRate::BitsPerSec(kTargetBitrateBps),
5113 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005114 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005115 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005116 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005117
5118 // Trigger initial configuration.
5119 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005120 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5121 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
sprangfda496a2017-06-15 04:21:07 -07005122 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
sprangfda496a2017-06-15 04:21:07 -07005123 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005124 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005125 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005126 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005127
Niels Möller7dc26b72017-12-06 10:27:48 +01005128 EXPECT_EQ(
5129 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5130 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005131
5132 // Trigger overuse, max framerate should be reduced.
5133 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5134 stats.input_frame_rate = kFramerate;
5135 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005136 video_stream_encoder_->TriggerCpuOveruse();
5137 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005138 int adapted_framerate =
5139 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005140 EXPECT_LT(adapted_framerate, kFramerate);
5141
5142 // Change degradation preference to not enable framerate scaling. Target
5143 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005144 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005145 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005146 EXPECT_EQ(
5147 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5148 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005149
mflodmancc3d4422017-08-03 08:27:51 -07005150 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005151}
5152
mflodmancc3d4422017-08-03 08:27:51 -07005153TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005154 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005155 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005156 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5157 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5158 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005159 const int kWidth = 640;
5160 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005161
asaperssonfab67072017-04-04 05:51:49 -07005162 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005163
5164 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005165 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005166
5167 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005168 EXPECT_TRUE_WAIT(
5169 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005170
sprangc5d62e22017-04-02 23:53:04 -07005171 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005172
asaperssonfab67072017-04-04 05:51:49 -07005173 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005174 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005175 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005176
5177 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005178 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005179
Henrik Boström2671dac2020-05-19 16:29:09 +02005180 EXPECT_TRUE_WAIT(
5181 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005182
mflodmancc3d4422017-08-03 08:27:51 -07005183 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005184}
5185
mflodmancc3d4422017-08-03 08:27:51 -07005186TEST_F(VideoStreamEncoderTest,
5187 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005188 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005190 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5191 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5192 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005193 const int kWidth = 640;
5194 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005195
5196 // We expect the n initial frames to get dropped.
5197 int i;
5198 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005199 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005200 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005201 }
5202 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005203 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005204 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005205
5206 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005207 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005208
mflodmancc3d4422017-08-03 08:27:51 -07005209 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005210}
5211
mflodmancc3d4422017-08-03 08:27:51 -07005212TEST_F(VideoStreamEncoderTest,
5213 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005214 const int kWidth = 640;
5215 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005217 DataRate::BitsPerSec(kLowTargetBitrateBps),
5218 DataRate::BitsPerSec(kLowTargetBitrateBps),
5219 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005220
5221 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005222 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005223 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005224
asaperssonfab67072017-04-04 05:51:49 -07005225 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005226 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005227 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005228
mflodmancc3d4422017-08-03 08:27:51 -07005229 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005230}
5231
mflodmancc3d4422017-08-03 08:27:51 -07005232TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005233 const int kWidth = 640;
5234 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005235 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005236
5237 VideoEncoderConfig video_encoder_config;
5238 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5239 // Make format different, to force recreation of encoder.
5240 video_encoder_config.video_format.parameters["foo"] = "foo";
5241 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005242 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005244 DataRate::BitsPerSec(kLowTargetBitrateBps),
5245 DataRate::BitsPerSec(kLowTargetBitrateBps),
5246 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005247
kthelgasonb83797b2017-02-14 11:57:25 -08005248 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005249 video_stream_encoder_->SetSource(&video_source_,
5250 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005251
asaperssonfab67072017-04-04 05:51:49 -07005252 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005253 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005254 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005255
mflodmancc3d4422017-08-03 08:27:51 -07005256 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005257 fake_encoder_.SetQualityScaling(true);
5258}
5259
Åsa Persson139f4dc2019-08-02 09:29:58 +02005260TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5261 webrtc::test::ScopedFieldTrials field_trials(
5262 "WebRTC-Video-QualityScalerSettings/"
5263 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5264 // Reset encoder for field trials to take effect.
5265 ConfigureEncoder(video_encoder_config_.Copy());
5266 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5267 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5268 const int kWidth = 640;
5269 const int kHeight = 360;
5270
Henrik Boström381d1092020-05-12 18:49:07 +02005271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005272 DataRate::BitsPerSec(kTargetBitrateBps),
5273 DataRate::BitsPerSec(kTargetBitrateBps),
5274 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005275 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5276 // Frame should not be dropped.
5277 WaitForEncodedFrame(1);
5278
Henrik Boström381d1092020-05-12 18:49:07 +02005279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005280 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5281 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5282 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005283 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5284 // Frame should not be dropped.
5285 WaitForEncodedFrame(2);
5286
Henrik Boström381d1092020-05-12 18:49:07 +02005287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005288 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5289 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5290 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005291 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5292 // Expect to drop this frame, the wait should time out.
5293 ExpectDroppedFrame();
5294
5295 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005296 EXPECT_TRUE_WAIT(
5297 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005298 video_stream_encoder_->Stop();
5299}
5300
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005301TEST_F(VideoStreamEncoderTest,
5302 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5303 webrtc::test::ScopedFieldTrials field_trials(
5304 "WebRTC-Video-QualityScalerSettings/"
5305 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5306 fake_encoder_.SetQualityScaling(false);
5307 ConfigureEncoder(video_encoder_config_.Copy());
5308 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.2;
5309 const int kTooLowBitrateForFrameSizeBps = kTargetBitrateBps * 0.19;
5310 const int kWidth = 640;
5311 const int kHeight = 360;
5312
5313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5314 DataRate::BitsPerSec(kTargetBitrateBps),
5315 DataRate::BitsPerSec(kTargetBitrateBps),
5316 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5317 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5318 // Frame should not be dropped.
5319 WaitForEncodedFrame(1);
5320
5321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5322 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5323 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5324 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5325 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5326 // Frame should not be dropped.
5327 WaitForEncodedFrame(2);
5328
5329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5330 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5331 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5332 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5333 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5334 // Not dropped since quality scaling is disabled.
5335 WaitForEncodedFrame(3);
5336
5337 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005339 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5340
5341 video_stream_encoder_->Stop();
5342}
5343
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005344TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
5345 const int kLowTargetBitrateBps = 400000;
5346 // Set simulcast.
5347 ResetEncoder("VP8", 3, 1, 1, false);
5348 fake_encoder_.SetQualityScaling(true);
5349 const int kWidth = 1280;
5350 const int kHeight = 720;
5351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5352 DataRate::BitsPerSec(kLowTargetBitrateBps),
5353 DataRate::BitsPerSec(kLowTargetBitrateBps),
5354 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5355 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5356 // Frame should not be dropped.
5357 WaitForEncodedFrame(1);
5358
5359 // Trigger QVGA "singlecast"
5360 // Update the config.
5361 VideoEncoderConfig video_encoder_config;
5362 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5363 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005364 video_encoder_config.video_stream_factory =
5365 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
5366 "VP8", /*max qp*/ 56, /*screencast*/ false,
5367 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005368 for (auto& layer : video_encoder_config.simulcast_layers) {
5369 layer.num_temporal_layers = 1;
5370 layer.max_framerate = kDefaultFramerate;
5371 }
5372 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5373 video_encoder_config.content_type =
5374 VideoEncoderConfig::ContentType::kRealtimeVideo;
5375
5376 video_encoder_config.simulcast_layers[0].active = true;
5377 video_encoder_config.simulcast_layers[1].active = false;
5378 video_encoder_config.simulcast_layers[2].active = false;
5379
5380 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5381 kMaxPayloadLength);
5382 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5383
5384 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5385 // Frame should not be dropped.
5386 WaitForEncodedFrame(2);
5387
5388 // Trigger HD "singlecast"
5389 video_encoder_config.simulcast_layers[0].active = false;
5390 video_encoder_config.simulcast_layers[1].active = false;
5391 video_encoder_config.simulcast_layers[2].active = true;
5392
5393 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5394 kMaxPayloadLength);
5395 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5396
5397 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5398 // Frame should be dropped because of initial frame drop.
5399 ExpectDroppedFrame();
5400
5401 // Expect the sink_wants to specify a scaled frame.
5402 EXPECT_TRUE_WAIT(
5403 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5404 video_stream_encoder_->Stop();
5405}
5406
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005407TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
5408 const int kLowTargetBitrateBps = 400000;
5409 // Set simulcast.
5410 ResetEncoder("VP9", 1, 1, 3, false);
5411 fake_encoder_.SetQualityScaling(true);
5412 const int kWidth = 1280;
5413 const int kHeight = 720;
5414 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5415 DataRate::BitsPerSec(kLowTargetBitrateBps),
5416 DataRate::BitsPerSec(kLowTargetBitrateBps),
5417 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5418 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5419 // Frame should not be dropped.
5420 WaitForEncodedFrame(1);
5421
5422 // Trigger QVGA "singlecast"
5423 // Update the config.
5424 VideoEncoderConfig video_encoder_config;
5425 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5426 &video_encoder_config);
5427 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5428 vp9_settings.numberOfSpatialLayers = 3;
5429 // Since only one layer is active - automatic resize should be enabled.
5430 vp9_settings.automaticResizeOn = true;
5431 video_encoder_config.encoder_specific_settings =
5432 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5433 vp9_settings);
5434 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5435 video_encoder_config.content_type =
5436 VideoEncoderConfig::ContentType::kRealtimeVideo;
5437 // Currently simulcast layers |active| flags are used to inidicate
5438 // which SVC layers are active.
5439 video_encoder_config.simulcast_layers.resize(3);
5440
5441 video_encoder_config.simulcast_layers[0].active = true;
5442 video_encoder_config.simulcast_layers[1].active = false;
5443 video_encoder_config.simulcast_layers[2].active = false;
5444
5445 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5446 kMaxPayloadLength);
5447 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5448
5449 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5450 // Frame should not be dropped.
5451 WaitForEncodedFrame(2);
5452
5453 // Trigger HD "singlecast"
5454 video_encoder_config.simulcast_layers[0].active = false;
5455 video_encoder_config.simulcast_layers[1].active = false;
5456 video_encoder_config.simulcast_layers[2].active = true;
5457
5458 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5459 kMaxPayloadLength);
5460 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5461
5462 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5463 // Frame should be dropped because of initial frame drop.
5464 ExpectDroppedFrame();
5465
5466 // Expect the sink_wants to specify a scaled frame.
5467 EXPECT_TRUE_WAIT(
5468 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5469 video_stream_encoder_->Stop();
5470}
5471
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005472TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005473 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5474 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5475 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5476 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5477 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5478 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5479 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5480 fake_encoder_.SetResolutionBitrateLimits(
5481 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5482
5483 VideoEncoderConfig video_encoder_config;
5484 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5485 &video_encoder_config);
5486 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5487 vp9_settings.numberOfSpatialLayers = 3;
5488 // Since only one layer is active - automatic resize should be enabled.
5489 vp9_settings.automaticResizeOn = true;
5490 video_encoder_config.encoder_specific_settings =
5491 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5492 vp9_settings);
5493 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5494 video_encoder_config.content_type =
5495 VideoEncoderConfig::ContentType::kRealtimeVideo;
5496 // Simulcast layers are used to indicate which spatial layers are active.
5497 video_encoder_config.simulcast_layers.resize(3);
5498 video_encoder_config.simulcast_layers[0].active = false;
5499 video_encoder_config.simulcast_layers[1].active = true;
5500 video_encoder_config.simulcast_layers[2].active = false;
5501
5502 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5503 kMaxPayloadLength);
5504 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5505
5506 // The encoder bitrate limits for 360p should be used.
5507 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5508 EXPECT_FALSE(WaitForFrame(1000));
5509 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5510 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5511 VideoCodecType::kVideoCodecVP9);
5512 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5513 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5514 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5515 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5516 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
5517 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5518 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
5519 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5520
5521 // The encoder bitrate limits for 270p should be used.
5522 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5523 EXPECT_FALSE(WaitForFrame(1000));
5524 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5525 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5526 VideoCodecType::kVideoCodecVP9);
5527 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5528 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5529 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5530 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5531 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
5532 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5533 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
5534 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5535
5536 video_stream_encoder_->Stop();
5537}
5538
5539TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005540 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5541 VideoEncoderConfig video_encoder_config;
5542 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5543 &video_encoder_config);
5544 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5545 vp9_settings.numberOfSpatialLayers = 3;
5546 // Since only one layer is active - automatic resize should be enabled.
5547 vp9_settings.automaticResizeOn = true;
5548 video_encoder_config.encoder_specific_settings =
5549 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5550 vp9_settings);
5551 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5552 video_encoder_config.content_type =
5553 VideoEncoderConfig::ContentType::kRealtimeVideo;
5554 // Simulcast layers are used to indicate which spatial layers are active.
5555 video_encoder_config.simulcast_layers.resize(3);
5556 video_encoder_config.simulcast_layers[0].active = false;
5557 video_encoder_config.simulcast_layers[1].active = true;
5558 video_encoder_config.simulcast_layers[2].active = false;
5559
5560 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5561 kMaxPayloadLength);
5562 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5563
5564 // The default bitrate limits for 360p should be used.
5565 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005566 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5567 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005568 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5569 EXPECT_FALSE(WaitForFrame(1000));
5570 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5571 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5572 VideoCodecType::kVideoCodecVP9);
5573 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5574 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5575 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5576 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5577 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
5578 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5579 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5580 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5581
5582 // The default bitrate limits for 270p should be used.
5583 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005584 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5585 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005586 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5587 EXPECT_FALSE(WaitForFrame(1000));
5588 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5589 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5590 VideoCodecType::kVideoCodecVP9);
5591 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5592 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5593 EXPECT_EQ(480, fake_encoder_.video_codec().spatialLayers[0].width);
5594 EXPECT_EQ(270, fake_encoder_.video_codec().spatialLayers[0].height);
5595 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
5596 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5597 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
5598 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5599
5600 video_stream_encoder_->Stop();
5601}
5602
5603TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5604 webrtc::test::ScopedFieldTrials field_trials(
5605 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5606 VideoEncoderConfig video_encoder_config;
5607 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5608 &video_encoder_config);
5609 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5610 vp9_settings.numberOfSpatialLayers = 3;
5611 // Since only one layer is active - automatic resize should be enabled.
5612 vp9_settings.automaticResizeOn = true;
5613 video_encoder_config.encoder_specific_settings =
5614 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5615 vp9_settings);
5616 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5617 video_encoder_config.content_type =
5618 VideoEncoderConfig::ContentType::kRealtimeVideo;
5619 // Simulcast layers are used to indicate which spatial layers are active.
5620 video_encoder_config.simulcast_layers.resize(3);
5621 video_encoder_config.simulcast_layers[0].active = false;
5622 video_encoder_config.simulcast_layers[1].active = true;
5623 video_encoder_config.simulcast_layers[2].active = false;
5624
5625 // Reset encoder for field trials to take effect.
5626 ConfigureEncoder(video_encoder_config.Copy());
5627
5628 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5629 kMaxPayloadLength);
5630 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5631
5632 // The default bitrate limits for 360p should not be used.
5633 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005634 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5635 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005636 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5637 EXPECT_FALSE(WaitForFrame(1000));
5638 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
Sergey Silkina86b29b2021-03-05 13:29:19 +01005639 EXPECT_EQ(fake_encoder_.video_codec().codecType, kVideoCodecVP9);
Åsa Persson258e9892021-02-25 10:39:51 +01005640 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 2);
5641 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5642 EXPECT_EQ(640, fake_encoder_.video_codec().spatialLayers[0].width);
5643 EXPECT_EQ(360, fake_encoder_.video_codec().spatialLayers[0].height);
5644 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
5645 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5646
5647 video_stream_encoder_->Stop();
5648}
5649
5650TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5651 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5652 /*num_spatial_layers=*/1, /*screenshare=*/false);
5653
5654 // The default singlecast bitrate limits for 720p should not be used.
5655 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005656 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5657 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005658 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5659 EXPECT_FALSE(WaitForFrame(1000));
5660 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5661 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5662 VideoCodecType::kVideoCodecVP9);
5663 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 1);
5664 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5665 EXPECT_EQ(1280, fake_encoder_.video_codec().spatialLayers[0].width);
5666 EXPECT_EQ(720, fake_encoder_.video_codec().spatialLayers[0].height);
5667 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
5668 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5669
5670 video_stream_encoder_->Stop();
5671}
5672
5673TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005674 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5675 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5676 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5677 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5678 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5679 fake_encoder_.SetResolutionBitrateLimits(
5680 {kEncoderLimits180p, kEncoderLimits720p});
5681
5682 VideoEncoderConfig video_encoder_config;
5683 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5684 &video_encoder_config);
5685 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5686 vp9_settings.numberOfSpatialLayers = 3;
5687 // Since only one layer is active - automatic resize should be enabled.
5688 vp9_settings.automaticResizeOn = true;
5689 video_encoder_config.encoder_specific_settings =
5690 new rtc::RefCountedObject<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
5691 vp9_settings);
5692 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrateBps;
5693 video_encoder_config.content_type =
5694 VideoEncoderConfig::ContentType::kRealtimeVideo;
5695 // Simulcast layers are used to indicate which spatial layers are active.
5696 video_encoder_config.simulcast_layers.resize(3);
5697 video_encoder_config.simulcast_layers[0].active = true;
5698 video_encoder_config.simulcast_layers[1].active = false;
5699 video_encoder_config.simulcast_layers[2].active = false;
5700
5701 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5702 kMaxPayloadLength);
5703 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5704
5705 // Limits not applied on lowest stream, limits for 180p should not be used.
5706 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5707 EXPECT_FALSE(WaitForFrame(1000));
5708 EXPECT_EQ(fake_encoder_.video_codec().numberOfSimulcastStreams, 1);
5709 EXPECT_EQ(fake_encoder_.video_codec().codecType,
5710 VideoCodecType::kVideoCodecVP9);
5711 EXPECT_EQ(fake_encoder_.video_codec().VP9()->numberOfSpatialLayers, 3);
5712 EXPECT_TRUE(fake_encoder_.video_codec().spatialLayers[0].active);
5713 EXPECT_EQ(320, fake_encoder_.video_codec().spatialLayers[0].width);
5714 EXPECT_EQ(180, fake_encoder_.video_codec().spatialLayers[0].height);
5715 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
5716 fake_encoder_.video_codec().spatialLayers[0].minBitrate * 1000);
5717 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
5718 fake_encoder_.video_codec().spatialLayers[0].maxBitrate * 1000);
5719
5720 video_stream_encoder_->Stop();
5721}
5722
5723TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005724 InitialFrameDropActivatesWhenResolutionIncreases) {
5725 const int kWidth = 640;
5726 const int kHeight = 360;
5727
5728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5729 DataRate::BitsPerSec(kTargetBitrateBps),
5730 DataRate::BitsPerSec(kTargetBitrateBps),
5731 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5732 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5733 // Frame should not be dropped.
5734 WaitForEncodedFrame(1);
5735
5736 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5737 DataRate::BitsPerSec(kLowTargetBitrateBps),
5738 DataRate::BitsPerSec(kLowTargetBitrateBps),
5739 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5740 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5741 // Frame should not be dropped, bitrate not too low for frame.
5742 WaitForEncodedFrame(2);
5743
5744 // Incoming resolution increases.
5745 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5746 // Expect to drop this frame, bitrate too low for frame.
5747 ExpectDroppedFrame();
5748
5749 // Expect the sink_wants to specify a scaled frame.
5750 EXPECT_TRUE_WAIT(
5751 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5752 video_stream_encoder_->Stop();
5753}
5754
5755TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5756 const int kWidth = 640;
5757 const int kHeight = 360;
5758 // So that quality scaling doesn't happen by itself.
5759 fake_encoder_.SetQp(kQpHigh);
5760
5761 AdaptingFrameForwarder source(&time_controller_);
5762 source.set_adaptation_enabled(true);
5763 video_stream_encoder_->SetSource(
5764 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5765
5766 int timestamp = 1;
5767
5768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5769 DataRate::BitsPerSec(kTargetBitrateBps),
5770 DataRate::BitsPerSec(kTargetBitrateBps),
5771 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5772 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5773 WaitForEncodedFrame(timestamp);
5774 timestamp += 9000;
5775 // Long pause to disable all first BWE drop logic.
5776 AdvanceTime(TimeDelta::Millis(1000));
5777
5778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5779 DataRate::BitsPerSec(kLowTargetBitrateBps),
5780 DataRate::BitsPerSec(kLowTargetBitrateBps),
5781 DataRate::BitsPerSec(kLowTargetBitrateBps), 0, 0, 0);
5782 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5783 // Not dropped frame, as initial frame drop is disabled by now.
5784 WaitForEncodedFrame(timestamp);
5785 timestamp += 9000;
5786 AdvanceTime(TimeDelta::Millis(100));
5787
5788 // Quality adaptation down.
5789 video_stream_encoder_->TriggerQualityLow();
5790
5791 // Adaptation has an effect.
5792 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5793 5000);
5794
5795 // Frame isn't dropped as initial frame dropper is disabled.
5796 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5797 WaitForEncodedFrame(timestamp);
5798 timestamp += 9000;
5799 AdvanceTime(TimeDelta::Millis(100));
5800
5801 // Quality adaptation up.
5802 video_stream_encoder_->TriggerQualityHigh();
5803
5804 // Adaptation has an effect.
5805 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5806 5000);
5807
5808 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5809 // Frame should not be dropped, as initial framedropper is off.
5810 WaitForEncodedFrame(timestamp);
5811
5812 video_stream_encoder_->Stop();
5813}
5814
Åsa Persson7f354f82021-02-04 15:52:15 +01005815TEST_F(VideoStreamEncoderTest,
5816 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5817 const int kMinStartBps360p = 222000;
5818 fake_encoder_.SetResolutionBitrateLimits(
5819 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5820 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5821 800000)});
5822
5823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5824 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5825 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5826 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5827 0, 0, 0);
5828 // Frame should not be dropped, bitrate not too low for frame.
5829 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5830 WaitForEncodedFrame(1);
5831
5832 // Incoming resolution increases, initial frame drop activates.
5833 // Frame should be dropped, link allocation too low for frame.
5834 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5835 ExpectDroppedFrame();
5836
5837 // Expect sink_wants to specify a scaled frame.
5838 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5839 5000);
5840 video_stream_encoder_->Stop();
5841}
5842
5843TEST_F(VideoStreamEncoderTest,
5844 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
5845 const int kMinStartBps360p = 222000;
5846 fake_encoder_.SetResolutionBitrateLimits(
5847 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5848 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5849 800000)});
5850
5851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5852 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5853 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5854 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
5855 0, 0, 0);
5856 // Frame should not be dropped, bitrate not too low for frame.
5857 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5858 WaitForEncodedFrame(1);
5859
5860 // Incoming resolution increases, initial frame drop activates.
5861 // Frame should be dropped, link allocation not too low for frame.
5862 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5863 WaitForEncodedFrame(2);
5864
5865 video_stream_encoder_->Stop();
5866}
5867
Åsa Perssone644a032019-11-08 15:56:00 +01005868TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
5869 webrtc::test::ScopedFieldTrials field_trials(
5870 "WebRTC-Video-QualityRampupSettings/min_pixels:1,min_duration_ms:2000/");
5871
5872 // Reset encoder for field trials to take effect.
5873 VideoEncoderConfig config = video_encoder_config_.Copy();
5874 config.max_bitrate_bps = kTargetBitrateBps;
Evan Shrubsoledff79252020-04-16 11:34:32 +02005875 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01005876 ConfigureEncoder(std::move(config));
5877 fake_encoder_.SetQp(kQpLow);
5878
5879 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005880 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01005881 source.set_adaptation_enabled(true);
5882 video_stream_encoder_->SetSource(&source,
5883 DegradationPreference::MAINTAIN_FRAMERATE);
5884
5885 // Start at low bitrate.
5886 const int kLowBitrateBps = 200000;
Henrik Boström381d1092020-05-12 18:49:07 +02005887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5888 DataRate::BitsPerSec(kLowBitrateBps),
5889 DataRate::BitsPerSec(kLowBitrateBps),
5890 DataRate::BitsPerSec(kLowBitrateBps), 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005891
5892 // Expect first frame to be dropped and resolution to be limited.
5893 const int kWidth = 1280;
5894 const int kHeight = 720;
5895 const int64_t kFrameIntervalMs = 100;
5896 int64_t timestamp_ms = kFrameIntervalMs;
5897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5898 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02005899 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5900 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01005901
5902 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02005903 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5904 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01005905
5906 // Insert frames and advance |min_duration_ms|.
5907 for (size_t i = 1; i <= 10; i++) {
5908 timestamp_ms += kFrameIntervalMs;
5909 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5910 WaitForEncodedFrame(timestamp_ms);
5911 }
5912 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5913 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
5914
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005915 AdvanceTime(TimeDelta::Millis(2000));
Åsa Perssone644a032019-11-08 15:56:00 +01005916
5917 // Insert frame should trigger high BW and release quality limitation.
5918 timestamp_ms += kFrameIntervalMs;
5919 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5920 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02005921 // The ramp-up code involves the adaptation queue, give it time to execute.
5922 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02005923 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005924 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01005925
5926 // Frame should not be adapted.
5927 timestamp_ms += kFrameIntervalMs;
5928 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5929 WaitForEncodedFrame(kWidth, kHeight);
5930 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5931
5932 video_stream_encoder_->Stop();
5933}
5934
mflodmancc3d4422017-08-03 08:27:51 -07005935TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005936 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01005937 webrtc::test::ScopedFieldTrials field_trials(
5938 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005939 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005940 source.set_adaptation_enabled(true);
5941 video_stream_encoder_->SetSource(&source,
5942 DegradationPreference::MAINTAIN_FRAMERATE);
5943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5944 DataRate::BitsPerSec(kTargetBitrateBps),
5945 DataRate::BitsPerSec(kTargetBitrateBps),
5946 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
5947 fake_encoder_.SetQp(kQpHigh + 1);
5948 const int kWidth = 1280;
5949 const int kHeight = 720;
5950 const int64_t kFrameIntervalMs = 100;
5951 int64_t timestamp_ms = kFrameIntervalMs;
5952 for (size_t i = 1; i <= 100; i++) {
5953 timestamp_ms += kFrameIntervalMs;
5954 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5955 WaitForEncodedFrame(timestamp_ms);
5956 }
5957 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
5958 // for the first time.
5959 // TODO(eshr): We should avoid these waits by using threads with simulated
5960 // time.
5961 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
5962 2000 * 2.5 * 2);
5963 timestamp_ms += kFrameIntervalMs;
5964 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
5965 WaitForEncodedFrame(timestamp_ms);
5966 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5967 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
5968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
5969
5970 // Disable Quality scaling by turning off scaler on the encoder and
5971 // reconfiguring.
5972 fake_encoder_.SetQualityScaling(false);
5973 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
5974 kMaxPayloadLength);
5975 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02005976 AdvanceTime(TimeDelta::Millis(0));
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02005977 // Since we turned off the quality scaler, the adaptations made by it are
5978 // removed.
5979 EXPECT_THAT(source.sink_wants(), ResolutionMax());
5980 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
5981
5982 video_stream_encoder_->Stop();
5983}
5984
5985TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07005986 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
5987 const int kTooSmallWidth = 10;
5988 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02005989 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005990 DataRate::BitsPerSec(kTargetBitrateBps),
5991 DataRate::BitsPerSec(kTargetBitrateBps),
5992 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07005993
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005994 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07005995 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005996 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005997 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02005998 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07005999 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6000
6001 // Trigger adapt down, too small frame, expect no change.
6002 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006003 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006004 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006005 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006006 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6007 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6008
mflodmancc3d4422017-08-03 08:27:51 -07006009 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006010}
6011
mflodmancc3d4422017-08-03 08:27:51 -07006012TEST_F(VideoStreamEncoderTest,
6013 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006014 const int kTooSmallWidth = 10;
6015 const int kTooSmallHeight = 10;
6016 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006017 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006018 DataRate::BitsPerSec(kTargetBitrateBps),
6019 DataRate::BitsPerSec(kTargetBitrateBps),
6020 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006021
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006022 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006023 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006024 video_stream_encoder_->SetSource(&source,
6025 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006026 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006027 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6028 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6029
6030 // Trigger adapt down, expect limited framerate.
6031 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006032 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006033 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006034 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6036 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6037 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6038
6039 // Trigger adapt down, too small frame, expect no change.
6040 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006041 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006042 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006043 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006044 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6045 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6046 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6047
mflodmancc3d4422017-08-03 08:27:51 -07006048 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006049}
6050
mflodmancc3d4422017-08-03 08:27:51 -07006051TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006052 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006054 DataRate::BitsPerSec(kTargetBitrateBps),
6055 DataRate::BitsPerSec(kTargetBitrateBps),
6056 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006057 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006058 const int kFrameWidth = 1280;
6059 const int kFrameHeight = 720;
6060 video_source_.IncomingCapturedFrame(
6061 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006062 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006063 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006064}
6065
sprangb1ca0732017-02-01 08:38:12 -08006066// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006067TEST_F(VideoStreamEncoderTest,
6068 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006070 DataRate::BitsPerSec(kTargetBitrateBps),
6071 DataRate::BitsPerSec(kTargetBitrateBps),
6072 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006073
6074 const int kFrameWidth = 1280;
6075 const int kFrameHeight = 720;
6076 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006077 // requested by
6078 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006079 video_source_.set_adaptation_enabled(true);
6080
6081 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006082 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006083 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006084
6085 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006086 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006087 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006088 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006089 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006090
asaperssonfab67072017-04-04 05:51:49 -07006091 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006092 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006093 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006094 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006095 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006096
mflodmancc3d4422017-08-03 08:27:51 -07006097 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006098}
sprangfe627f32017-03-29 08:24:59 -07006099
mflodmancc3d4422017-08-03 08:27:51 -07006100TEST_F(VideoStreamEncoderTest,
6101 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006102 const int kFrameWidth = 1280;
6103 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006104
Henrik Boström381d1092020-05-12 18:49:07 +02006105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006106 DataRate::BitsPerSec(kTargetBitrateBps),
6107 DataRate::BitsPerSec(kTargetBitrateBps),
6108 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006109 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006110 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006111 video_source_.set_adaptation_enabled(true);
6112
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006113 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006114
6115 video_source_.IncomingCapturedFrame(
6116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006117 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006118
6119 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006120 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006121
6122 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006123 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006124 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006125 video_source_.IncomingCapturedFrame(
6126 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006127 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006128 }
6129
6130 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006131 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006132 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006133 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006134 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006135 video_source_.IncomingCapturedFrame(
6136 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006137 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006138 ++num_frames_dropped;
6139 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006140 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006141 }
6142 }
6143
sprang4847ae62017-06-27 07:06:52 -07006144 // Add some slack to account for frames dropped by the frame dropper.
6145 const int kErrorMargin = 1;
6146 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006147 kErrorMargin);
6148
6149 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006150 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006151 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006152 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006153 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006154 video_source_.IncomingCapturedFrame(
6155 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006156 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006157 ++num_frames_dropped;
6158 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006159 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006160 }
6161 }
sprang4847ae62017-06-27 07:06:52 -07006162 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006163 kErrorMargin);
6164
6165 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006166 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006167 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006168 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006169 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006170 video_source_.IncomingCapturedFrame(
6171 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006172 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006173 ++num_frames_dropped;
6174 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006175 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006176 }
6177 }
sprang4847ae62017-06-27 07:06:52 -07006178 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006179 kErrorMargin);
6180
6181 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006182 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006183 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006184 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006185 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006186 video_source_.IncomingCapturedFrame(
6187 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006188 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006189 ++num_frames_dropped;
6190 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006191 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006192 }
6193 }
6194 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6195
mflodmancc3d4422017-08-03 08:27:51 -07006196 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006197}
6198
mflodmancc3d4422017-08-03 08:27:51 -07006199TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006200 const int kFramerateFps = 5;
6201 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006202 const int kFrameWidth = 1280;
6203 const int kFrameHeight = 720;
6204
sprang4847ae62017-06-27 07:06:52 -07006205 // Reconfigure encoder with two temporal layers and screensharing, which will
6206 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006207 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006208
Henrik Boström381d1092020-05-12 18:49:07 +02006209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006210 DataRate::BitsPerSec(kTargetBitrateBps),
6211 DataRate::BitsPerSec(kTargetBitrateBps),
6212 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006213 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006214 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006215 video_source_.set_adaptation_enabled(true);
6216
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006217 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006218
6219 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006220 rtc::VideoSinkWants last_wants;
6221 do {
6222 last_wants = video_source_.sink_wants();
6223
sprangc5d62e22017-04-02 23:53:04 -07006224 // Insert frames to get a new fps estimate...
6225 for (int j = 0; j < kFramerateFps; ++j) {
6226 video_source_.IncomingCapturedFrame(
6227 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006228 if (video_source_.last_sent_width()) {
6229 sink_.WaitForEncodedFrame(timestamp_ms);
6230 }
sprangc5d62e22017-04-02 23:53:04 -07006231 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006232 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006233 }
6234 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006235 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006236 } while (video_source_.sink_wants().max_framerate_fps <
6237 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006238
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006239 EXPECT_THAT(video_source_.sink_wants(),
6240 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006241
mflodmancc3d4422017-08-03 08:27:51 -07006242 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006243}
asaperssonf7e294d2017-06-13 23:25:22 -07006244
mflodmancc3d4422017-08-03 08:27:51 -07006245TEST_F(VideoStreamEncoderTest,
6246 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006247 const int kWidth = 1280;
6248 const int kHeight = 720;
6249 const int64_t kFrameIntervalMs = 150;
6250 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006252 DataRate::BitsPerSec(kTargetBitrateBps),
6253 DataRate::BitsPerSec(kTargetBitrateBps),
6254 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006255
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006256 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006257 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006258 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006259 video_stream_encoder_->SetSource(&source,
6260 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006261 timestamp_ms += kFrameIntervalMs;
6262 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006263 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006264 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006265 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6266 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6267 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6268
6269 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006270 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006271 timestamp_ms += kFrameIntervalMs;
6272 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006273 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006274 EXPECT_THAT(source.sink_wants(),
6275 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006276 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6277 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6278 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6279
6280 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006281 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006282 timestamp_ms += kFrameIntervalMs;
6283 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006285 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006286 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6287 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6288 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6289
6290 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006292 timestamp_ms += kFrameIntervalMs;
6293 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006294 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006295 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006296 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6298 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6299
6300 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006302 timestamp_ms += kFrameIntervalMs;
6303 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006304 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006305 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6309
6310 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006312 timestamp_ms += kFrameIntervalMs;
6313 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006314 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006315 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6317 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6318 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6319
6320 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006321 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006322 timestamp_ms += kFrameIntervalMs;
6323 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006324 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006325 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006326 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6328 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6329
6330 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006331 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006332 timestamp_ms += kFrameIntervalMs;
6333 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006334 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006335 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006336 rtc::VideoSinkWants last_wants = source.sink_wants();
6337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6338 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6339 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6340
6341 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006342 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006343 timestamp_ms += kFrameIntervalMs;
6344 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006345 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006346 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006347 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6348 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6349 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6350
Evan Shrubsole64469032020-06-11 10:45:29 +02006351 // Trigger adapt up, expect expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006352 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006353 timestamp_ms += kFrameIntervalMs;
6354 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006355 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006356 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006357 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6358 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6359 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6360
6361 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006363 timestamp_ms += kFrameIntervalMs;
6364 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006365 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006366 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006367 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6368 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6369 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6370
6371 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006373 timestamp_ms += kFrameIntervalMs;
6374 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006375 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006376 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006377 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6378 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6379 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6380
6381 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006382 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006383 timestamp_ms += kFrameIntervalMs;
6384 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006385 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006386 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006387 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6388 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6389 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6390
6391 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006392 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006393 timestamp_ms += kFrameIntervalMs;
6394 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006395 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006396 EXPECT_THAT(source.sink_wants(), FpsMax());
6397 EXPECT_EQ(source.sink_wants().max_pixel_count,
6398 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006399 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6401 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6402
6403 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006404 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006405 timestamp_ms += kFrameIntervalMs;
6406 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006407 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006408 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006409 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6410 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6411 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6412
Åsa Persson30ab0152019-08-27 12:22:33 +02006413 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006415 timestamp_ms += kFrameIntervalMs;
6416 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006417 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006418 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006420 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6421 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6422 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6423
6424 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006425 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006426 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006427 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6428
mflodmancc3d4422017-08-03 08:27:51 -07006429 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006430}
6431
mflodmancc3d4422017-08-03 08:27:51 -07006432TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006433 const int kWidth = 1280;
6434 const int kHeight = 720;
6435 const int64_t kFrameIntervalMs = 150;
6436 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006438 DataRate::BitsPerSec(kTargetBitrateBps),
6439 DataRate::BitsPerSec(kTargetBitrateBps),
6440 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006441
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006442 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006443 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006444 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006445 video_stream_encoder_->SetSource(&source,
6446 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006447 timestamp_ms += kFrameIntervalMs;
6448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006449 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006450 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6453 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6454 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6455 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6457
6458 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006459 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006460 timestamp_ms += kFrameIntervalMs;
6461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006462 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006463 EXPECT_THAT(source.sink_wants(),
6464 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6467 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6468 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6469 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6470 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6471
6472 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006473 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006474 timestamp_ms += kFrameIntervalMs;
6475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006476 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006477 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6481 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6482 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6483 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6484
6485 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006486 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006487 timestamp_ms += kFrameIntervalMs;
6488 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006489 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006490 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006492 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6493 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6494 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6495 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6496 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6497
Evan Shrubsole64469032020-06-11 10:45:29 +02006498 // Trigger cpu adapt up, expect no change since QP is most limited.
6499 {
6500 // Store current sink wants since we expect no change and if there is no
6501 // change then last_wants() is not updated.
6502 auto previous_sink_wants = source.sink_wants();
6503 video_stream_encoder_->TriggerCpuUnderuse();
6504 timestamp_ms += kFrameIntervalMs;
6505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6506 WaitForEncodedFrame(timestamp_ms);
6507 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6508 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6509 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6510 }
6511
6512 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6513 video_stream_encoder_->TriggerQualityHigh();
6514 timestamp_ms += kFrameIntervalMs;
6515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6516 WaitForEncodedFrame(timestamp_ms);
6517 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6520 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6521 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6522 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6523 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6524
6525 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6526 // expect increased resolution (960x540@30fps).
6527 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006528 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006529 timestamp_ms += kFrameIntervalMs;
6530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006531 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006532 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6535 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6536 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6537 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006538 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006539
Evan Shrubsole64469032020-06-11 10:45:29 +02006540 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6541 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006542 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006543 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006544 timestamp_ms += kFrameIntervalMs;
6545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006546 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6550 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6551 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6553 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006554 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006555
6556 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006557 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006559 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006560 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006561
mflodmancc3d4422017-08-03 08:27:51 -07006562 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006563}
6564
mflodmancc3d4422017-08-03 08:27:51 -07006565TEST_F(VideoStreamEncoderTest,
6566 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006567 const int kWidth = 640;
6568 const int kHeight = 360;
6569 const int kFpsLimit = 15;
6570 const int64_t kFrameIntervalMs = 150;
6571 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006573 DataRate::BitsPerSec(kTargetBitrateBps),
6574 DataRate::BitsPerSec(kTargetBitrateBps),
6575 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006576
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006577 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006578 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006579 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006580 video_stream_encoder_->SetSource(&source,
6581 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006582 timestamp_ms += kFrameIntervalMs;
6583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006584 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006585 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6589 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6590 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6591 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6592
6593 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006594 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006595 timestamp_ms += kFrameIntervalMs;
6596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006597 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006598 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006599 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6600 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6601 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6603 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6604 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6605
6606 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006607 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006608 timestamp_ms += kFrameIntervalMs;
6609 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006610 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006611 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006614 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6615 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6616 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6617 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6618
Evan Shrubsole64469032020-06-11 10:45:29 +02006619 // Trigger cpu adapt up, expect no change because quality is most limited.
6620 {
6621 auto previous_sink_wants = source.sink_wants();
6622 // Store current sink wants since we expect no change ind if there is no
6623 // change then last__wants() is not updated.
6624 video_stream_encoder_->TriggerCpuUnderuse();
6625 timestamp_ms += kFrameIntervalMs;
6626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6627 WaitForEncodedFrame(timestamp_ms);
6628 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6629 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6630 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6631 }
6632
6633 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6634 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006635 timestamp_ms += kFrameIntervalMs;
6636 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006637 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006638 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006639 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6641 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006642 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6643 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6644 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006645
Evan Shrubsole64469032020-06-11 10:45:29 +02006646 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006647 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006648 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006649 timestamp_ms += kFrameIntervalMs;
6650 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006651 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006652 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6654 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6655 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6656 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6657 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006658 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006659
6660 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006661 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006662 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006663 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006664 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006665
mflodmancc3d4422017-08-03 08:27:51 -07006666 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006667}
6668
mflodmancc3d4422017-08-03 08:27:51 -07006669TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006670 const int kFrameWidth = 1920;
6671 const int kFrameHeight = 1080;
6672 // 3/4 of 1920.
6673 const int kAdaptedFrameWidth = 1440;
6674 // 3/4 of 1080 rounded down to multiple of 4.
6675 const int kAdaptedFrameHeight = 808;
6676 const int kFramerate = 24;
6677
Henrik Boström381d1092020-05-12 18:49:07 +02006678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006679 DataRate::BitsPerSec(kTargetBitrateBps),
6680 DataRate::BitsPerSec(kTargetBitrateBps),
6681 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006682 // Trigger reconfigure encoder (without resetting the entire instance).
6683 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006684 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6685 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
ilnik6b826ef2017-06-16 06:53:48 -07006686 video_encoder_config.max_bitrate_bps = kTargetBitrateBps;
ilnik6b826ef2017-06-16 06:53:48 -07006687 video_encoder_config.video_stream_factory =
Åsa Persson17b29b92020-10-17 12:57:58 +02006688 new rtc::RefCountedObject<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006689 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006690 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006691 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006692
6693 video_source_.set_adaptation_enabled(true);
6694
6695 video_source_.IncomingCapturedFrame(
6696 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006697 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006698
6699 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006700 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006701 video_source_.IncomingCapturedFrame(
6702 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006703 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006704
mflodmancc3d4422017-08-03 08:27:51 -07006705 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006706}
6707
mflodmancc3d4422017-08-03 08:27:51 -07006708TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006709 const int kFrameWidth = 1280;
6710 const int kFrameHeight = 720;
6711 const int kLowFps = 2;
6712 const int kHighFps = 30;
6713
Henrik Boström381d1092020-05-12 18:49:07 +02006714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006715 DataRate::BitsPerSec(kTargetBitrateBps),
6716 DataRate::BitsPerSec(kTargetBitrateBps),
6717 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006718
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006719 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006720 max_framerate_ = kLowFps;
6721
6722 // Insert 2 seconds of 2fps video.
6723 for (int i = 0; i < kLowFps * 2; ++i) {
6724 video_source_.IncomingCapturedFrame(
6725 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6726 WaitForEncodedFrame(timestamp_ms);
6727 timestamp_ms += 1000 / kLowFps;
6728 }
6729
6730 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006732 DataRate::BitsPerSec(kTargetBitrateBps),
6733 DataRate::BitsPerSec(kTargetBitrateBps),
6734 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006735 video_source_.IncomingCapturedFrame(
6736 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6737 WaitForEncodedFrame(timestamp_ms);
6738 timestamp_ms += 1000 / kLowFps;
6739
6740 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6741
6742 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006743 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006744 const int kFrameIntervalMs = 1000 / kHighFps;
6745 max_framerate_ = kHighFps;
6746 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6747 video_source_.IncomingCapturedFrame(
6748 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6749 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6750 // be dropped if the encoder hans't been updated with the new higher target
6751 // framerate yet, causing it to overshoot the target bitrate and then
6752 // suffering the wrath of the media optimizer.
6753 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6754 timestamp_ms += kFrameIntervalMs;
6755 }
6756
6757 // Don expect correct measurement just yet, but it should be higher than
6758 // before.
6759 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6760
mflodmancc3d4422017-08-03 08:27:51 -07006761 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006762}
6763
mflodmancc3d4422017-08-03 08:27:51 -07006764TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006765 const int kFrameWidth = 1280;
6766 const int kFrameHeight = 720;
6767 const int kTargetBitrateBps = 1000000;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006768 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006769 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006770 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006771
Henrik Boström381d1092020-05-12 18:49:07 +02006772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006773 DataRate::BitsPerSec(kTargetBitrateBps),
6774 DataRate::BitsPerSec(kTargetBitrateBps),
6775 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006776 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006777
6778 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006779 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006780 video_source_.IncomingCapturedFrame(
6781 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6782 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006783 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006784
6785 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6787 DataRate::BitsPerSec(0), DataRate::BitsPerSec(0), DataRate::BitsPerSec(0),
6788 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006789
6790 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006791 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006792 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006793
Per Kjellanderdcef6412020-10-07 15:09:05 +02006794 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006795 video_source_.IncomingCapturedFrame(
6796 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6797 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006798 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006799
mflodmancc3d4422017-08-03 08:27:51 -07006800 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006801}
ilnik6b826ef2017-06-16 06:53:48 -07006802
Niels Möller4db138e2018-04-19 09:04:13 +02006803TEST_F(VideoStreamEncoderTest,
6804 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6805 const int kFrameWidth = 1280;
6806 const int kFrameHeight = 720;
6807 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006809 DataRate::BitsPerSec(kTargetBitrateBps),
6810 DataRate::BitsPerSec(kTargetBitrateBps),
6811 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006812 video_source_.IncomingCapturedFrame(
6813 CreateFrame(1, kFrameWidth, kFrameHeight));
6814 WaitForEncodedFrame(1);
6815 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6816 .low_encode_usage_threshold_percent,
6817 default_options.low_encode_usage_threshold_percent);
6818 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6819 .high_encode_usage_threshold_percent,
6820 default_options.high_encode_usage_threshold_percent);
6821 video_stream_encoder_->Stop();
6822}
6823
6824TEST_F(VideoStreamEncoderTest,
6825 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6826 const int kFrameWidth = 1280;
6827 const int kFrameHeight = 720;
6828 CpuOveruseOptions hardware_options;
6829 hardware_options.low_encode_usage_threshold_percent = 150;
6830 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006831 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006832
Henrik Boström381d1092020-05-12 18:49:07 +02006833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006834 DataRate::BitsPerSec(kTargetBitrateBps),
6835 DataRate::BitsPerSec(kTargetBitrateBps),
6836 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006837 video_source_.IncomingCapturedFrame(
6838 CreateFrame(1, kFrameWidth, kFrameHeight));
6839 WaitForEncodedFrame(1);
6840 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6841 .low_encode_usage_threshold_percent,
6842 hardware_options.low_encode_usage_threshold_percent);
6843 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6844 .high_encode_usage_threshold_percent,
6845 hardware_options.high_encode_usage_threshold_percent);
6846 video_stream_encoder_->Stop();
6847}
6848
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006849TEST_F(VideoStreamEncoderTest,
6850 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6851 const int kFrameWidth = 1280;
6852 const int kFrameHeight = 720;
6853
6854 const CpuOveruseOptions default_options;
6855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6856 DataRate::BitsPerSec(kTargetBitrateBps),
6857 DataRate::BitsPerSec(kTargetBitrateBps),
6858 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
6859 video_source_.IncomingCapturedFrame(
6860 CreateFrame(1, kFrameWidth, kFrameHeight));
6861 WaitForEncodedFrame(1);
6862 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6863 .low_encode_usage_threshold_percent,
6864 default_options.low_encode_usage_threshold_percent);
6865 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6866 .high_encode_usage_threshold_percent,
6867 default_options.high_encode_usage_threshold_percent);
6868
6869 CpuOveruseOptions hardware_options;
6870 hardware_options.low_encode_usage_threshold_percent = 150;
6871 hardware_options.high_encode_usage_threshold_percent = 200;
6872 fake_encoder_.SetIsHardwareAccelerated(true);
6873
6874 video_source_.IncomingCapturedFrame(
6875 CreateFrame(2, kFrameWidth, kFrameHeight));
6876 WaitForEncodedFrame(2);
6877
6878 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6879 .low_encode_usage_threshold_percent,
6880 hardware_options.low_encode_usage_threshold_percent);
6881 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6882 .high_encode_usage_threshold_percent,
6883 hardware_options.high_encode_usage_threshold_percent);
6884
6885 video_stream_encoder_->Stop();
6886}
6887
Niels Möller6bb5ab92019-01-11 11:11:10 +01006888TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
6889 const int kFrameWidth = 320;
6890 const int kFrameHeight = 240;
6891 const int kFps = 30;
6892 const int kTargetBitrateBps = 120000;
6893 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
6894
Henrik Boström381d1092020-05-12 18:49:07 +02006895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006896 DataRate::BitsPerSec(kTargetBitrateBps),
6897 DataRate::BitsPerSec(kTargetBitrateBps),
6898 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006899
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006900 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006901 max_framerate_ = kFps;
6902
6903 // Insert 3 seconds of video, verify number of drops with normal bitrate.
6904 fake_encoder_.SimulateOvershoot(1.0);
6905 int num_dropped = 0;
6906 for (int i = 0; i < kNumFramesInRun; ++i) {
6907 video_source_.IncomingCapturedFrame(
6908 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6909 // Wait up to two frame durations for a frame to arrive.
6910 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6911 ++num_dropped;
6912 }
6913 timestamp_ms += 1000 / kFps;
6914 }
6915
Erik Språnga8d48ab2019-02-08 14:17:40 +01006916 // Framerate should be measured to be near the expected target rate.
6917 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6918
6919 // Frame drops should be within 5% of expected 0%.
6920 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006921
6922 // Make encoder produce frames at double the expected bitrate during 3 seconds
6923 // of video, verify number of drops. Rate needs to be slightly changed in
6924 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01006925 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02006926 const RateControlSettings trials =
6927 RateControlSettings::ParseFromFieldTrials();
6928 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01006929 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02006930 // frame dropping since the adjuter will try to just lower the target
6931 // bitrate rather than drop frames. If network headroom can be used, it
6932 // doesn't push back as hard so we don't need quite as much overshoot.
6933 // These numbers are unfortunately a bit magical but there's not trivial
6934 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01006935 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01006936 }
6937 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02006938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006939 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6940 DataRate::BitsPerSec(kTargetBitrateBps + 1000),
6941 DataRate::BitsPerSec(kTargetBitrateBps + 1000), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006942 num_dropped = 0;
6943 for (int i = 0; i < kNumFramesInRun; ++i) {
6944 video_source_.IncomingCapturedFrame(
6945 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6946 // Wait up to two frame durations for a frame to arrive.
6947 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
6948 ++num_dropped;
6949 }
6950 timestamp_ms += 1000 / kFps;
6951 }
6952
Henrik Boström381d1092020-05-12 18:49:07 +02006953 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006954 DataRate::BitsPerSec(kTargetBitrateBps),
6955 DataRate::BitsPerSec(kTargetBitrateBps),
6956 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01006957
6958 // Target framerate should be still be near the expected target, despite
6959 // the frame drops.
6960 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
6961
6962 // Frame drops should be within 5% of expected 50%.
6963 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006964
6965 video_stream_encoder_->Stop();
6966}
6967
6968TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
6969 const int kFrameWidth = 320;
6970 const int kFrameHeight = 240;
6971 const int kActualInputFps = 24;
6972 const int kTargetBitrateBps = 120000;
6973
6974 ASSERT_GT(max_framerate_, kActualInputFps);
6975
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006976 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01006977 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02006978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01006979 DataRate::BitsPerSec(kTargetBitrateBps),
6980 DataRate::BitsPerSec(kTargetBitrateBps),
6981 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01006982
6983 // Insert 3 seconds of video, with an input fps lower than configured max.
6984 for (int i = 0; i < kActualInputFps * 3; ++i) {
6985 video_source_.IncomingCapturedFrame(
6986 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6987 // Wait up to two frame durations for a frame to arrive.
6988 WaitForEncodedFrame(timestamp_ms);
6989 timestamp_ms += 1000 / kActualInputFps;
6990 }
6991
6992 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
6993
6994 video_stream_encoder_->Stop();
6995}
6996
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006997TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01006998 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02006999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007000 DataRate::BitsPerSec(kTargetBitrateBps),
7001 DataRate::BitsPerSec(kTargetBitrateBps),
7002 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007003
7004 fake_encoder_.BlockNextEncode();
7005 video_source_.IncomingCapturedFrame(
7006 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7007 WaitForEncodedFrame(1);
7008 // On the very first frame full update should be forced.
7009 rect = fake_encoder_.GetLastUpdateRect();
7010 EXPECT_EQ(rect.offset_x, 0);
7011 EXPECT_EQ(rect.offset_y, 0);
7012 EXPECT_EQ(rect.height, codec_height_);
7013 EXPECT_EQ(rect.width, codec_width_);
7014 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7015 // call to ContinueEncode.
7016 video_source_.IncomingCapturedFrame(
7017 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7018 ExpectDroppedFrame();
7019 video_source_.IncomingCapturedFrame(
7020 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7021 ExpectDroppedFrame();
7022 fake_encoder_.ContinueEncode();
7023 WaitForEncodedFrame(3);
7024 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7025 rect = fake_encoder_.GetLastUpdateRect();
7026 EXPECT_EQ(rect.offset_x, 1);
7027 EXPECT_EQ(rect.offset_y, 0);
7028 EXPECT_EQ(rect.width, 10);
7029 EXPECT_EQ(rect.height, 1);
7030
7031 video_source_.IncomingCapturedFrame(
7032 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7033 WaitForEncodedFrame(4);
7034 // Previous frame was encoded, so no accumulation should happen.
7035 rect = fake_encoder_.GetLastUpdateRect();
7036 EXPECT_EQ(rect.offset_x, 0);
7037 EXPECT_EQ(rect.offset_y, 0);
7038 EXPECT_EQ(rect.width, 1);
7039 EXPECT_EQ(rect.height, 1);
7040
7041 video_stream_encoder_->Stop();
7042}
7043
Erik Språngd7329ca2019-02-21 21:19:53 +01007044TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007046 DataRate::BitsPerSec(kTargetBitrateBps),
7047 DataRate::BitsPerSec(kTargetBitrateBps),
7048 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007049
7050 // First frame is always keyframe.
7051 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7052 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007053 EXPECT_THAT(
7054 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007055 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007056
7057 // Insert delta frame.
7058 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7059 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007060 EXPECT_THAT(
7061 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007062 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007063
7064 // Request next frame be a key-frame.
7065 video_stream_encoder_->SendKeyFrame();
7066 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7067 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007068 EXPECT_THAT(
7069 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007070 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007071
7072 video_stream_encoder_->Stop();
7073}
7074
7075TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7076 // Setup simulcast with three streams.
7077 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007079 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7080 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7081 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007082 // Wait for all three layers before triggering event.
7083 sink_.SetNumExpectedLayers(3);
7084
7085 // First frame is always keyframe.
7086 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7087 WaitForEncodedFrame(1);
7088 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007089 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7090 VideoFrameType::kVideoFrameKey,
7091 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007092
7093 // Insert delta frame.
7094 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7095 WaitForEncodedFrame(2);
7096 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007097 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7098 VideoFrameType::kVideoFrameDelta,
7099 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007100
7101 // Request next frame be a key-frame.
7102 // Only first stream is configured to produce key-frame.
7103 video_stream_encoder_->SendKeyFrame();
7104 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7105 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007106
7107 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7108 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007109 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007110 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007111 VideoFrameType::kVideoFrameKey,
7112 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007113
7114 video_stream_encoder_->Stop();
7115}
7116
7117TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7118 // Configure internal source factory and setup test again.
7119 encoder_factory_.SetHasInternalSource(true);
7120 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007122 DataRate::BitsPerSec(kTargetBitrateBps),
7123 DataRate::BitsPerSec(kTargetBitrateBps),
7124 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007125
7126 // Call encoder directly, simulating internal source where encoded frame
7127 // callback in VideoStreamEncoder is called despite no OnFrame().
7128 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7129 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007130 EXPECT_THAT(
7131 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007132 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007133
Niels Möller8f7ce222019-03-21 15:43:58 +01007134 const std::vector<VideoFrameType> kDeltaFrame = {
7135 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007136 // Need to set timestamp manually since manually for injected frame.
7137 VideoFrame frame = CreateFrame(101, nullptr);
7138 frame.set_timestamp(101);
7139 fake_encoder_.InjectFrame(frame, false);
7140 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007141 EXPECT_THAT(
7142 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007143 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007144
7145 // Request key-frame. The forces a dummy frame down into the encoder.
7146 fake_encoder_.ExpectNullFrame();
7147 video_stream_encoder_->SendKeyFrame();
7148 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007149 EXPECT_THAT(
7150 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007151 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007152
7153 video_stream_encoder_->Stop();
7154}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007155
7156TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7157 // Configure internal source factory and setup test again.
7158 encoder_factory_.SetHasInternalSource(true);
7159 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007161 DataRate::BitsPerSec(kTargetBitrateBps),
7162 DataRate::BitsPerSec(kTargetBitrateBps),
7163 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007164
7165 int64_t timestamp = 1;
7166 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007167 image.capture_time_ms_ = ++timestamp;
7168 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7169 const int64_t kEncodeFinishDelayMs = 10;
7170 image.timing_.encode_start_ms = timestamp;
7171 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007172 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007173 // Wait for frame without incrementing clock.
7174 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7175 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7176 // capture timestamp should be kEncodeFinishDelayMs in the past.
7177 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007178 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007179
7180 video_stream_encoder_->Stop();
7181}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007182
7183TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007184 // SPS contains VUI with restrictions on the maximum number of reordered
7185 // pictures, there is no need to rewrite the bitstream to enable faster
7186 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007187 ResetEncoder("H264", 1, 1, 1, false);
7188
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7190 DataRate::BitsPerSec(kTargetBitrateBps),
7191 DataRate::BitsPerSec(kTargetBitrateBps),
7192 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7193 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007194
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007195 fake_encoder_.SetEncodedImageData(
7196 EncodedImageBuffer::Create(optimal_sps, sizeof(optimal_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007197
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007198 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7199 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007200
7201 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7202 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007203
7204 video_stream_encoder_->Stop();
7205}
7206
7207TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007208 // SPS does not contain VUI, the bitstream is will be rewritten with added
7209 // VUI with restrictions on the maximum number of reordered pictures to
7210 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007211 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7212 0x00, 0x00, 0x03, 0x03, 0xF4,
7213 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007214 ResetEncoder("H264", 1, 1, 1, false);
7215
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7217 DataRate::BitsPerSec(kTargetBitrateBps),
7218 DataRate::BitsPerSec(kTargetBitrateBps),
7219 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007221
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007222 fake_encoder_.SetEncodedImageData(
7223 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007224
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007225 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7226 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007227
7228 EXPECT_THAT(sink_.GetLastEncodedImageData(),
7229 testing::ElementsAreArray(optimal_sps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007230
7231 video_stream_encoder_->Stop();
7232}
7233
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007234TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7235 const int kFrameWidth = 1280;
7236 const int kFrameHeight = 720;
7237 const int kTargetBitrateBps = 300000; // To low for HD resolution.
7238
Henrik Boström381d1092020-05-12 18:49:07 +02007239 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007240 DataRate::BitsPerSec(kTargetBitrateBps),
7241 DataRate::BitsPerSec(kTargetBitrateBps),
7242 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007243 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7244
7245 // Insert a first video frame. It should be dropped because of downscale in
7246 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007247 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007248 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7249 frame.set_rotation(kVideoRotation_270);
7250 video_source_.IncomingCapturedFrame(frame);
7251
7252 ExpectDroppedFrame();
7253
7254 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007255 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007256 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7257 frame.set_rotation(kVideoRotation_90);
7258 video_source_.IncomingCapturedFrame(frame);
7259
7260 WaitForEncodedFrame(timestamp_ms);
7261 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7262
7263 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007264 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007265 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7266 frame.set_rotation(kVideoRotation_180);
7267 video_source_.IncomingCapturedFrame(frame);
7268
7269 WaitForEncodedFrame(timestamp_ms);
7270 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7271
7272 video_stream_encoder_->Stop();
7273}
7274
Erik Språng5056af02019-09-02 15:53:11 +02007275TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7276 const int kFrameWidth = 320;
7277 const int kFrameHeight = 180;
7278
7279 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007280 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007281 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7282 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7283 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007284 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007285 /*rtt_ms=*/0,
7286 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007287
7288 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007289 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007290 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7291 frame.set_rotation(kVideoRotation_270);
7292 video_source_.IncomingCapturedFrame(frame);
7293 WaitForEncodedFrame(timestamp_ms);
7294
7295 // Set a target rate below the minimum allowed by the codec settings.
7296 VideoCodec codec_config = fake_encoder_.codec_config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007297 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7298 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007300 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007301 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007302 /*link_allocation=*/target_rate,
7303 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007304 /*rtt_ms=*/0,
7305 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007306 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7307
7308 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7309 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7310 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007311 DataRate allocation_sum =
7312 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007313 EXPECT_EQ(min_rate, allocation_sum);
7314 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7315
7316 video_stream_encoder_->Stop();
7317}
7318
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007319TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007321 DataRate::BitsPerSec(kTargetBitrateBps),
7322 DataRate::BitsPerSec(kTargetBitrateBps),
7323 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007324 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007325 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007326 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7327 WaitForEncodedFrame(1);
7328
7329 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7330 ASSERT_TRUE(prev_rate_settings.has_value());
7331 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7332 kDefaultFramerate);
7333
7334 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7335 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7336 timestamp_ms += 1000 / kDefaultFramerate;
7337 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7338 WaitForEncodedFrame(timestamp_ms);
7339 }
7340 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7341 kDefaultFramerate);
7342 // Capture larger frame to trigger a reconfigure.
7343 codec_height_ *= 2;
7344 codec_width_ *= 2;
7345 timestamp_ms += 1000 / kDefaultFramerate;
7346 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7347 WaitForEncodedFrame(timestamp_ms);
7348
7349 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7350 auto current_rate_settings =
7351 fake_encoder_.GetAndResetLastRateControlSettings();
7352 // Ensure we have actually reconfigured twice
7353 // The rate settings should have been set again even though
7354 // they haven't changed.
7355 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007356 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007357
7358 video_stream_encoder_->Stop();
7359}
7360
philipeld9cc8c02019-09-16 14:53:40 +02007361struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007362 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7363 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7364 MOCK_METHOD(void,
7365 RequestEncoderSwitch,
7366 (const webrtc::SdpVideoFormat& format),
7367 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007368};
7369
philipel9b058032020-02-10 11:30:00 +01007370TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7371 constexpr int kDontCare = 100;
7372 StrictMock<MockEncoderSelector> encoder_selector;
7373 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7374 &fake_encoder_, &encoder_selector);
7375 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7376
7377 // Reset encoder for new configuration to take effect.
7378 ConfigureEncoder(video_encoder_config_.Copy());
7379
7380 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7381
7382 video_source_.IncomingCapturedFrame(
7383 CreateFrame(kDontCare, kDontCare, kDontCare));
7384 video_stream_encoder_->Stop();
7385
7386 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7387 // to it's factory, so in order for the encoder instance in the
7388 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7389 // reset the |video_stream_encoder_| here.
7390 video_stream_encoder_.reset();
7391}
7392
7393TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7394 constexpr int kDontCare = 100;
7395
7396 NiceMock<MockEncoderSelector> encoder_selector;
7397 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7398 video_send_config_.encoder_settings.encoder_switch_request_callback =
7399 &switch_callback;
7400 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7401 &fake_encoder_, &encoder_selector);
7402 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7403
7404 // Reset encoder for new configuration to take effect.
7405 ConfigureEncoder(video_encoder_config_.Copy());
7406
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007407 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007408 .WillByDefault(Return(SdpVideoFormat("AV1")));
7409 EXPECT_CALL(switch_callback,
7410 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7411 Field(&SdpVideoFormat::name, "AV1"))));
7412
Henrik Boström381d1092020-05-12 18:49:07 +02007413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007414 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7415 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7416 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007417 /*fraction_lost=*/0,
7418 /*rtt_ms=*/0,
7419 /*cwnd_reduce_ratio=*/0);
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007420 AdvanceTime(TimeDelta::Millis(0));
philipel9b058032020-02-10 11:30:00 +01007421
7422 video_stream_encoder_->Stop();
7423}
7424
7425TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7426 constexpr int kSufficientBitrateToNotDrop = 1000;
7427 constexpr int kDontCare = 100;
7428
7429 NiceMock<MockVideoEncoder> video_encoder;
7430 NiceMock<MockEncoderSelector> encoder_selector;
7431 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7432 video_send_config_.encoder_settings.encoder_switch_request_callback =
7433 &switch_callback;
7434 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7435 &video_encoder, &encoder_selector);
7436 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7437
7438 // Reset encoder for new configuration to take effect.
7439 ConfigureEncoder(video_encoder_config_.Copy());
7440
7441 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7442 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7443 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007445 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7446 /*stable_target_bitrate=*/
7447 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7448 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007449 /*fraction_lost=*/0,
7450 /*rtt_ms=*/0,
7451 /*cwnd_reduce_ratio=*/0);
7452
7453 ON_CALL(video_encoder, Encode(_, _))
7454 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7455 ON_CALL(encoder_selector, OnEncoderBroken())
7456 .WillByDefault(Return(SdpVideoFormat("AV2")));
7457
7458 rtc::Event encode_attempted;
7459 EXPECT_CALL(switch_callback,
7460 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7461 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7462 EXPECT_EQ(format.name, "AV2");
7463 encode_attempted.Set();
7464 });
7465
7466 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7467 encode_attempted.Wait(3000);
7468
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007469 AdvanceTime(TimeDelta::Millis(0));
7470
philipel9b058032020-02-10 11:30:00 +01007471 video_stream_encoder_->Stop();
7472
7473 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7474 // to it's factory, so in order for the encoder instance in the
7475 // |video_stream_encoder_| to be destroyed before the |encoder_factory| we
7476 // reset the |video_stream_encoder_| here.
7477 video_stream_encoder_.reset();
7478}
7479
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007480TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007481 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007482 const int kFrameWidth = 320;
7483 const int kFrameHeight = 180;
7484
7485 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007486 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007488 /*target_bitrate=*/rate,
7489 /*stable_target_bitrate=*/rate,
7490 /*link_allocation=*/rate,
7491 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007492 /*rtt_ms=*/0,
7493 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007494
7495 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007496 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007497 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7498 frame.set_rotation(kVideoRotation_270);
7499 video_source_.IncomingCapturedFrame(frame);
7500 WaitForEncodedFrame(timestamp_ms);
7501 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7502
7503 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007504 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007506 /*target_bitrate=*/new_stable_rate,
7507 /*stable_target_bitrate=*/new_stable_rate,
7508 /*link_allocation=*/rate,
7509 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007510 /*rtt_ms=*/0,
7511 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007512 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7513 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7514 video_stream_encoder_->Stop();
7515}
7516
7517TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007518 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007519 const int kFrameWidth = 320;
7520 const int kFrameHeight = 180;
7521
7522 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007523 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007524 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007525 /*target_bitrate=*/rate,
7526 /*stable_target_bitrate=*/rate,
7527 /*link_allocation=*/rate,
7528 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007529 /*rtt_ms=*/0,
7530 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007531
7532 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007533 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007534 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7535 frame.set_rotation(kVideoRotation_270);
7536 video_source_.IncomingCapturedFrame(frame);
7537 WaitForEncodedFrame(timestamp_ms);
7538 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7539
7540 // Set a higher target rate without changing the link_allocation. Should not
7541 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007542 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007543 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007544 /*target_bitrate=*/rate,
7545 /*stable_target_bitrate=*/new_stable_rate,
7546 /*link_allocation=*/rate,
7547 /*fraction_lost=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007548 /*rtt_ms=*/0,
7549 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007550 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7551 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7552 video_stream_encoder_->Stop();
7553}
7554
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007555TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7556 test::ScopedFieldTrials field_trials(
7557 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7558 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7559 const int kFramerateFps = 30;
7560 const int kWidth = 1920;
7561 const int kHeight = 1080;
7562 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7563 // Works on screenshare mode.
7564 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7565 // We rely on the automatic resolution adaptation, but we handle framerate
7566 // adaptation manually by mocking the stats proxy.
7567 video_source_.set_adaptation_enabled(true);
7568
7569 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007571 DataRate::BitsPerSec(kTargetBitrateBps),
7572 DataRate::BitsPerSec(kTargetBitrateBps),
7573 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007574 video_stream_encoder_->SetSource(&video_source_,
7575 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007576 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007577
7578 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7579 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7580
7581 // Pass enough frames with the full update to trigger animation detection.
7582 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007583 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007584 frame.set_ntp_time_ms(timestamp_ms);
7585 frame.set_timestamp_us(timestamp_ms * 1000);
7586 video_source_.IncomingCapturedFrame(frame);
7587 WaitForEncodedFrame(timestamp_ms);
7588 }
7589
7590 // Resolution should be limited.
7591 rtc::VideoSinkWants expected;
7592 expected.max_framerate_fps = kFramerateFps;
7593 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007594 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007595
7596 // Pass one frame with no known update.
7597 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007598 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007599 frame.set_ntp_time_ms(timestamp_ms);
7600 frame.set_timestamp_us(timestamp_ms * 1000);
7601 frame.clear_update_rect();
7602
7603 video_source_.IncomingCapturedFrame(frame);
7604 WaitForEncodedFrame(timestamp_ms);
7605
7606 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007607 EXPECT_THAT(video_source_.sink_wants(),
7608 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007609
7610 video_stream_encoder_->Stop();
7611}
7612
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007613TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7614 const int kWidth = 720; // 540p adapted down.
7615 const int kHeight = 405;
7616 const int kNumFrames = 3;
7617 // Works on screenshare mode.
7618 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7619 /*num_spatial_layers=*/2, /*screenshare=*/true);
7620
7621 video_source_.set_adaptation_enabled(true);
7622
7623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7624 DataRate::BitsPerSec(kTargetBitrateBps),
7625 DataRate::BitsPerSec(kTargetBitrateBps),
7626 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7627
7628 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7629
7630 // Pass enough frames with the full update to trigger animation detection.
7631 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007632 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007633 frame.set_ntp_time_ms(timestamp_ms);
7634 frame.set_timestamp_us(timestamp_ms * 1000);
7635 video_source_.IncomingCapturedFrame(frame);
7636 WaitForEncodedFrame(timestamp_ms);
7637 }
7638
7639 video_stream_encoder_->Stop();
7640}
7641
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007642TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7643 const float downscale_factors[] = {4.0, 2.0, 1.0};
7644 const int number_layers =
7645 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7646 VideoEncoderConfig config;
7647 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7648 for (int i = 0; i < number_layers; ++i) {
7649 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7650 config.simulcast_layers[i].active = true;
7651 }
7652 config.video_stream_factory =
7653 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7654 "VP8", /*max qp*/ 56, /*screencast*/ false,
7655 /*screenshare enabled*/ false);
7656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7657 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7658 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7659 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7660
7661 // First initialization.
7662 // Encoder should be initialized. Next frame should be key frame.
7663 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7664 sink_.SetNumExpectedLayers(number_layers);
7665 int64_t timestamp_ms = kFrameIntervalMs;
7666 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7667 WaitForEncodedFrame(timestamp_ms);
7668 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7669 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7670 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7671 VideoFrameType::kVideoFrameKey,
7672 VideoFrameType::kVideoFrameKey}));
7673
7674 // Disable top layer.
7675 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7676 config.simulcast_layers[number_layers - 1].active = false;
7677 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7678 sink_.SetNumExpectedLayers(number_layers - 1);
7679 timestamp_ms += kFrameIntervalMs;
7680 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7681 WaitForEncodedFrame(timestamp_ms);
7682 EXPECT_EQ(1, fake_encoder_.GetNumEncoderInitializations());
7683 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7684 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7685 VideoFrameType::kVideoFrameDelta,
7686 VideoFrameType::kVideoFrameDelta}));
7687
7688 // Re-enable top layer.
7689 // Encoder should be re-initialized. Next frame should be key frame.
7690 config.simulcast_layers[number_layers - 1].active = true;
7691 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7692 sink_.SetNumExpectedLayers(number_layers);
7693 timestamp_ms += kFrameIntervalMs;
7694 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7695 WaitForEncodedFrame(timestamp_ms);
7696 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7697 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7698 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7699 VideoFrameType::kVideoFrameKey,
7700 VideoFrameType::kVideoFrameKey}));
7701
7702 // Top layer max rate change.
7703 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7704 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7705 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7706 sink_.SetNumExpectedLayers(number_layers);
7707 timestamp_ms += kFrameIntervalMs;
7708 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7709 WaitForEncodedFrame(timestamp_ms);
7710 EXPECT_EQ(2, fake_encoder_.GetNumEncoderInitializations());
7711 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7712 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7713 VideoFrameType::kVideoFrameDelta,
7714 VideoFrameType::kVideoFrameDelta}));
7715
7716 // Top layer resolution change.
7717 // Encoder should be re-initialized. Next frame should be key frame.
7718 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7719 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7720 sink_.SetNumExpectedLayers(number_layers);
7721 timestamp_ms += kFrameIntervalMs;
7722 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7723 WaitForEncodedFrame(timestamp_ms);
7724 EXPECT_EQ(3, fake_encoder_.GetNumEncoderInitializations());
7725 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7726 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7727 VideoFrameType::kVideoFrameKey,
7728 VideoFrameType::kVideoFrameKey}));
7729 video_stream_encoder_->Stop();
7730}
7731
Henrik Boström1124ed12021-02-25 10:30:39 +01007732TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7733 const int kFrameWidth = 1280;
7734 const int kFrameHeight = 720;
7735
7736 SetUp();
7737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7738 DataRate::BitsPerSec(kTargetBitrateBps),
7739 DataRate::BitsPerSec(kTargetBitrateBps),
7740 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7741
7742 // Capturing a frame should reconfigure the encoder and expose the encoder
7743 // resolution, which is the same as the input frame.
7744 int64_t timestamp_ms = kFrameIntervalMs;
7745 video_source_.IncomingCapturedFrame(
7746 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7747 WaitForEncodedFrame(timestamp_ms);
7748 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7749 EXPECT_THAT(video_source_.sink_wants().resolutions,
7750 ::testing::ElementsAreArray(
7751 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7752
7753 video_stream_encoder_->Stop();
7754}
7755
7756TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7757 // Pick downscale factors such that we never encode at full resolution - this
7758 // is an interesting use case. The frame resolution influences the encoder
7759 // resolutions, but if no layer has |scale_resolution_down_by| == 1 then the
7760 // encoder should not ask for the frame resolution. This allows video frames
7761 // to have the appearence of one resolution but optimize its internal buffers
7762 // for what is actually encoded.
7763 const size_t kNumSimulcastLayers = 3u;
7764 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7765 const int kFrameWidth = 1280;
7766 const int kFrameHeight = 720;
7767 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7768 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7769 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7770 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7771 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7772 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7773
7774 VideoEncoderConfig config;
7775 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7776 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7777 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7778 config.simulcast_layers[i].active = true;
7779 }
7780 config.video_stream_factory =
7781 new rtc::RefCountedObject<cricket::EncoderStreamFactory>(
7782 "VP8", /*max qp*/ 56, /*screencast*/ false,
7783 /*screenshare enabled*/ false);
7784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7785 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7786 DataRate::BitsPerSec(kSimulcastTargetBitrateBps),
7787 DataRate::BitsPerSec(kSimulcastTargetBitrateBps), 0, 0, 0);
7788
7789 // Capture a frame with all layers active.
7790 int64_t timestamp_ms = kFrameIntervalMs;
7791 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7792 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7793 video_source_.IncomingCapturedFrame(
7794 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7795 WaitForEncodedFrame(timestamp_ms);
7796 // Expect encoded resolutions to match the expected simulcast layers.
7797 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7798 EXPECT_THAT(
7799 video_source_.sink_wants().resolutions,
7800 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7801
7802 // Capture a frame with one of the layers inactive.
7803 timestamp_ms += kFrameIntervalMs;
7804 config.simulcast_layers[2].active = false;
7805 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7806 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7807 video_source_.IncomingCapturedFrame(
7808 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7809 WaitForEncodedFrame(timestamp_ms);
7810
7811 // Expect encoded resolutions to match the expected simulcast layers.
7812 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7813 EXPECT_THAT(video_source_.sink_wants().resolutions,
7814 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7815
7816 // Capture a frame with all but one layer turned off.
7817 timestamp_ms += kFrameIntervalMs;
7818 config.simulcast_layers[1].active = false;
7819 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7820 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7821 video_source_.IncomingCapturedFrame(
7822 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7823 WaitForEncodedFrame(timestamp_ms);
7824
7825 // Expect encoded resolutions to match the expected simulcast layers.
7826 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7827 EXPECT_THAT(video_source_.sink_wants().resolutions,
7828 ::testing::ElementsAreArray({kLayer0Size}));
7829
7830 video_stream_encoder_->Stop();
7831}
7832
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007833TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
7834 // Enable encoder source to force encoder reconfig.
7835 encoder_factory_.SetHasInternalSource(true);
7836 ResetEncoder("VP8", 1, 1, 1, false);
7837
7838 // Set QP on encoded frame and pass the frame to encode complete callback.
7839 // Since QP is present QP parsing won't be triggered and the original value
7840 // should be kept.
7841 EncodedImage encoded_image;
7842 encoded_image.qp_ = 123;
7843 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7844 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7845 CodecSpecificInfo codec_info;
7846 codec_info.codecType = kVideoCodecVP8;
7847 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7848 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7849 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7850 video_stream_encoder_->Stop();
7851}
7852
7853TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
7854 // Enable encoder source to force encoder reconfig.
7855 encoder_factory_.SetHasInternalSource(true);
7856 ResetEncoder("VP8", 1, 1, 1, false);
7857
7858 // Pass an encoded frame without QP to encode complete callback. QP should be
7859 // parsed and set.
7860 EncodedImage encoded_image;
7861 encoded_image.qp_ = -1;
7862 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7863 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7864 CodecSpecificInfo codec_info;
7865 codec_info.codecType = kVideoCodecVP8;
7866 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7867 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7868 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7869 video_stream_encoder_->Stop();
7870}
7871
7872TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7873 webrtc::test::ScopedFieldTrials field_trials(
7874 "WebRTC-QpParsingKillSwitch/Enabled/");
7875
7876 // Enable encoder source to force encoder reconfig.
7877 encoder_factory_.SetHasInternalSource(true);
7878 ResetEncoder("VP8", 1, 1, 1, false);
7879
7880 EncodedImage encoded_image;
7881 encoded_image.qp_ = -1;
7882 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7883 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7884 CodecSpecificInfo codec_info;
7885 codec_info.codecType = kVideoCodecVP8;
7886 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7887 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7888 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7889 video_stream_encoder_->Stop();
7890}
7891
Sergey Silkind19e3b92021-03-16 10:05:30 +00007892TEST_F(VideoStreamEncoderTest,
7893 QualityScalingNotAllowed_QualityScalingDisabled) {
7894 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7895
7896 // Disable scaling settings in encoder info.
7897 fake_encoder_.SetQualityScaling(false);
7898 // Disable quality scaling in encoder config.
7899 video_encoder_config.is_quality_scaling_allowed = false;
7900 ConfigureEncoder(std::move(video_encoder_config));
7901
7902 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7903 DataRate::BitsPerSec(kTargetBitrateBps),
7904 DataRate::BitsPerSec(kTargetBitrateBps),
7905 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7906
7907 test::FrameForwarder source;
7908 video_stream_encoder_->SetSource(
7909 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7910 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7911 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7912
7913 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7914 WaitForEncodedFrame(1);
7915 video_stream_encoder_->TriggerQualityLow();
7916 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7917
7918 video_stream_encoder_->Stop();
7919}
7920
7921#if !defined(WEBRTC_IOS)
7922// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
7923// disabled by default on iOS.
7924TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
7925 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7926
7927 // Disable scaling settings in encoder info.
7928 fake_encoder_.SetQualityScaling(false);
7929 // Enable quality scaling in encoder config.
7930 video_encoder_config.is_quality_scaling_allowed = true;
7931 ConfigureEncoder(std::move(video_encoder_config));
7932
7933 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7934 DataRate::BitsPerSec(kTargetBitrateBps),
7935 DataRate::BitsPerSec(kTargetBitrateBps),
7936 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
7937
7938 test::FrameForwarder source;
7939 video_stream_encoder_->SetSource(
7940 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7941 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7942 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7943
7944 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7945 WaitForEncodedFrame(1);
7946 video_stream_encoder_->TriggerQualityLow();
7947 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
7948
7949 video_stream_encoder_->Stop();
7950}
7951#endif
7952
Henrik Boström56db9ff2021-03-24 09:06:45 +01007953// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
7954class VideoStreamEncoderWithRealEncoderTest
7955 : public VideoStreamEncoderTest,
7956 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
7957 public:
7958 VideoStreamEncoderWithRealEncoderTest()
7959 : VideoStreamEncoderTest(),
7960 codec_type_(std::get<0>(GetParam())),
7961 allow_i420_conversion_(std::get<1>(GetParam())) {}
7962
7963 void SetUp() override {
7964 VideoStreamEncoderTest::SetUp();
7965 std::unique_ptr<VideoEncoder> encoder;
7966 switch (codec_type_) {
7967 case kVideoCodecVP8:
7968 encoder = VP8Encoder::Create();
7969 break;
7970 case kVideoCodecVP9:
7971 encoder = VP9Encoder::Create();
7972 break;
7973 case kVideoCodecAV1:
7974 encoder = CreateLibaomAv1Encoder();
7975 break;
7976 case kVideoCodecH264:
7977 encoder =
7978 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
7979 break;
7980 case kVideoCodecMultiplex:
7981 mock_encoder_factory_for_multiplex_ =
7982 std::make_unique<MockVideoEncoderFactory>();
7983 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
7984 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
7985 .WillRepeatedly([] { return VP8Encoder::Create(); });
7986 encoder = std::make_unique<MultiplexEncoderAdapter>(
7987 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
7988 false);
7989 break;
7990 default:
7991 RTC_NOTREACHED();
7992 }
7993 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
7994 }
7995
7996 void TearDown() override {
7997 video_stream_encoder_->Stop();
7998 // Ensure |video_stream_encoder_| is destroyed before
7999 // |encoder_proxy_factory_|.
8000 video_stream_encoder_.reset();
8001 VideoStreamEncoderTest::TearDown();
8002 }
8003
8004 protected:
8005 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8006 std::unique_ptr<VideoEncoder> encoder) {
8007 // Configure VSE to use the encoder.
8008 encoder_ = std::move(encoder);
8009 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8010 encoder_.get(), &encoder_selector_);
8011 video_send_config_.encoder_settings.encoder_factory =
8012 encoder_proxy_factory_.get();
8013 VideoEncoderConfig video_encoder_config;
8014 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8015 video_encoder_config_ = video_encoder_config.Copy();
8016 ConfigureEncoder(video_encoder_config_.Copy());
8017
8018 // Set bitrate to ensure frame is not dropped.
8019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8020 DataRate::BitsPerSec(kTargetBitrateBps),
8021 DataRate::BitsPerSec(kTargetBitrateBps),
8022 DataRate::BitsPerSec(kTargetBitrateBps), 0, 0, 0);
8023 }
8024
8025 const VideoCodecType codec_type_;
8026 const bool allow_i420_conversion_;
8027 NiceMock<MockEncoderSelector> encoder_selector_;
8028 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8029 std::unique_ptr<VideoEncoder> encoder_;
8030 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8031};
8032
8033TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8034 auto native_i420_frame = test::CreateMappableNativeFrame(
8035 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8036 video_source_.IncomingCapturedFrame(native_i420_frame);
8037 WaitForEncodedFrame(codec_width_, codec_height_);
8038
8039 auto mappable_native_buffer =
8040 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8041 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8042 mappable_native_buffer->GetMappedFramedBuffers();
8043 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8044 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8045 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8046 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8047}
8048
8049TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8050 auto native_nv12_frame = test::CreateMappableNativeFrame(
8051 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8052 video_source_.IncomingCapturedFrame(native_nv12_frame);
8053 WaitForEncodedFrame(codec_width_, codec_height_);
8054
8055 auto mappable_native_buffer =
8056 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8057 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8058 mappable_native_buffer->GetMappedFramedBuffers();
8059 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8060 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8061 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8062 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8063
8064 if (!allow_i420_conversion_) {
8065 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8066 }
8067}
8068
8069std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8070 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8071 VideoCodecType codec_type = std::get<0>(info.param);
8072 bool allow_i420_conversion = std::get<1>(info.param);
8073 std::string str;
8074 switch (codec_type) {
8075 case kVideoCodecGeneric:
8076 str = "Generic";
8077 break;
8078 case kVideoCodecVP8:
8079 str = "VP8";
8080 break;
8081 case kVideoCodecVP9:
8082 str = "VP9";
8083 break;
8084 case kVideoCodecAV1:
8085 str = "AV1";
8086 break;
8087 case kVideoCodecH264:
8088 str = "H264";
8089 break;
8090 case kVideoCodecMultiplex:
8091 str = "Multiplex";
8092 break;
8093 default:
8094 RTC_NOTREACHED();
8095 }
8096 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8097 return str;
8098}
8099
8100constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8101 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8102constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8103 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8104constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8105 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8106constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8107 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8108#if defined(WEBRTC_USE_H264)
8109constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8110 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8111
8112// The windows compiler does not tolerate #if statements inside the
8113// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8114// and without H264).
8115INSTANTIATE_TEST_SUITE_P(
8116 All,
8117 VideoStreamEncoderWithRealEncoderTest,
8118 ::testing::Values(kVP8DisallowConversion,
8119 kVP9DisallowConversion,
8120 kAV1AllowConversion,
8121 kMultiplexDisallowConversion,
8122 kH264AllowConversion),
8123 TestParametersVideoCodecAndAllowI420ConversionToString);
8124#else
8125INSTANTIATE_TEST_SUITE_P(
8126 All,
8127 VideoStreamEncoderWithRealEncoderTest,
8128 ::testing::Values(kVP8DisallowConversion,
8129 kVP9DisallowConversion,
8130 kAV1AllowConversion,
8131 kMultiplexDisallowConversion),
8132 TestParametersVideoCodecAndAllowI420ConversionToString);
8133#endif
8134
perkj26091b12016-09-01 01:17:40 -07008135} // namespace webrtc