blob: 49e561937a1e8620f3ac7cb5fcd034ba04b9fcf7 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020020#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010021#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020022#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010023#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010024#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080025#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020027#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020028#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010029#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010030#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020031#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020032#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010033#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020034#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010035#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020036#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070037#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080038#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020039#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010040#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
41#include "modules/video_coding/codecs/h264/include/h264.h"
42#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
43#include "modules/video_coding/codecs/vp8/include/vp8.h"
44#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020045#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020046#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020047#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010048#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020049#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010050#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020051#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020052#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080053#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020054#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010055#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020056#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020057#include "test/encoder_settings.h"
58#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020059#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010060#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020061#include "test/gmock.h"
62#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010063#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020064#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020065#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010066#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020067#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070068
69namespace webrtc {
70
sprang57c2fff2017-01-16 06:24:02 -080071using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020072using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020073using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020074using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020075using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020076using ::testing::Ge;
77using ::testing::Gt;
78using ::testing::Le;
79using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010080using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010081using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010085using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020086using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080087
perkj803d97f2016-11-01 11:45:46 -070088namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020089const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010090const int kQpLow = 1;
91const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020092const int kMinFramerateFps = 2;
93const int kMinBalancedFramerateFps = 7;
94const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080095const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020096const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
97const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
98const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
99const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800100const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700101const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200102const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200103const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200104const VideoEncoder::ResolutionBitrateLimits
105 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800108
Asa Persson606d3cb2021-10-04 10:07:11 +0200109uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200110 0x00, 0x00, 0x03, 0x03, 0xF4,
111 0x05, 0x03, 0xC7, 0xE0, 0x1B,
112 0x41, 0x10, 0x8D, 0x00};
113
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100114const uint8_t kCodedFrameVp8Qp25[] = {
115 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
116 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
117 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
118
perkj803d97f2016-11-01 11:45:46 -0700119class TestBuffer : public webrtc::I420Buffer {
120 public:
121 TestBuffer(rtc::Event* event, int width, int height)
122 : I420Buffer(width, height), event_(event) {}
123
124 private:
125 friend class rtc::RefCountedObject<TestBuffer>;
126 ~TestBuffer() override {
127 if (event_)
128 event_->Set();
129 }
130 rtc::Event* const event_;
131};
132
Henrik Boström56db9ff2021-03-24 09:06:45 +0100133// A fake native buffer that can't be converted to I420. Upon scaling, it
134// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700135class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
136 public:
137 FakeNativeBuffer(rtc::Event* event, int width, int height)
138 : event_(event), width_(width), height_(height) {}
139 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
140 int width() const override { return width_; }
141 int height() const override { return height_; }
142 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
143 return nullptr;
144 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100145 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
146 int offset_x,
147 int offset_y,
148 int crop_width,
149 int crop_height,
150 int scaled_width,
151 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200152 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
153 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100154 }
Noah Richards51db4212019-06-12 06:59:12 -0700155
156 private:
157 friend class rtc::RefCountedObject<FakeNativeBuffer>;
158 ~FakeNativeBuffer() override {
159 if (event_)
160 event_->Set();
161 }
162 rtc::Event* const event_;
163 const int width_;
164 const int height_;
165};
166
Evan Shrubsole895556e2020-10-05 09:15:13 +0200167// A fake native buffer that is backed by an NV12 buffer.
168class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
169 public:
170 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
171 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
172
173 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
174 int width() const override { return nv12_buffer_->width(); }
175 int height() const override { return nv12_buffer_->height(); }
176 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
177 return nv12_buffer_->ToI420();
178 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200179 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
180 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
181 if (absl::c_find(types, Type::kNV12) != types.end()) {
182 return nv12_buffer_;
183 }
184 return nullptr;
185 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200186 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
187
188 private:
189 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
190 ~FakeNV12NativeBuffer() override {
191 if (event_)
192 event_->Set();
193 }
194 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
195 rtc::Event* const event_;
196};
197
Niels Möller7dc26b72017-12-06 10:27:48 +0100198class CpuOveruseDetectorProxy : public OveruseFrameDetector {
199 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200200 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
201 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200202 last_target_framerate_fps_(-1),
203 framerate_updated_event_(true /* manual_reset */,
204 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100205 virtual ~CpuOveruseDetectorProxy() {}
206
207 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200208 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100209 last_target_framerate_fps_ = framerate_fps;
210 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200211 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100212 }
213
214 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200215 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100216 return last_target_framerate_fps_;
217 }
218
Niels Möller4db138e2018-04-19 09:04:13 +0200219 CpuOveruseOptions GetOptions() { return options_; }
220
Henrik Boström381d1092020-05-12 18:49:07 +0200221 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
222
Niels Möller7dc26b72017-12-06 10:27:48 +0100223 private:
Markus Handella3765182020-07-08 13:13:32 +0200224 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100225 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200226 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100227};
228
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200229class FakeVideoSourceRestrictionsListener
230 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200231 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200232 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200233 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200234 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200235 RTC_DCHECK(was_restrictions_updated_);
236 }
237
238 rtc::Event* restrictions_updated_event() {
239 return &restrictions_updated_event_;
240 }
241
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200242 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200243 void OnVideoSourceRestrictionsUpdated(
244 VideoSourceRestrictions restrictions,
245 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200246 rtc::scoped_refptr<Resource> reason,
247 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200248 was_restrictions_updated_ = true;
249 restrictions_updated_event_.Set();
250 }
251
252 private:
253 bool was_restrictions_updated_;
254 rtc::Event restrictions_updated_event_;
255};
256
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200257auto WantsFps(Matcher<int> fps_matcher) {
258 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
259 fps_matcher);
260}
261
262auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
263 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
264 AllOf(max_pixel_matcher, Gt(0)));
265}
266
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200267auto ResolutionMax() {
268 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200269 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200270 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
271 Eq(absl::nullopt)));
272}
273
274auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200275 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200276}
277
278auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200280}
281
282auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200284}
285
286auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200287 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200288}
289
290auto FpsMaxResolutionMax() {
291 return AllOf(FpsMax(), ResolutionMax());
292}
293
294auto UnlimitedSinkWants() {
295 return AllOf(FpsUnlimited(), ResolutionMax());
296}
297
298auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
299 Matcher<int> fps_range_matcher;
300
301 if (last_frame_pixels <= 320 * 240) {
302 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200303 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304 fps_range_matcher = AllOf(Ge(10), Le(15));
305 } else if (last_frame_pixels <= 640 * 480) {
306 fps_range_matcher = Ge(15);
307 } else {
308 fps_range_matcher = Eq(kDefaultFramerate);
309 }
310 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
311 fps_range_matcher);
312}
313
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200314auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
315 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
316 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
317}
318
319auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
320 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
321}
322
323auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
324 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
325}
326
327auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
328 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
329 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
330}
331
332auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
333 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
334 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
335}
336
337auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
338 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
339 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
340}
341
342auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
343 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
344 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
345}
346
mflodmancc3d4422017-08-03 08:27:51 -0700347class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700348 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200349 VideoStreamEncoderUnderTest(TimeController* time_controller,
350 TaskQueueFactory* task_queue_factory,
351 SendStatisticsProxy* stats_proxy,
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100352 const VideoStreamEncoderSettings& settings,
353 VideoStreamEncoder::BitrateAllocationCallbackType
354 allocation_callback_type)
Markus Handell28c71802021-11-08 10:11:55 +0100355 : VideoStreamEncoder(
356 time_controller->GetClock(),
357 1 /* number_of_cores */,
358 stats_proxy,
359 settings,
360 std::unique_ptr<OveruseFrameDetector>(
361 overuse_detector_proxy_ =
362 new CpuOveruseDetectorProxy(stats_proxy)),
363 FrameCadenceAdapterInterface::Create(TaskQueueBase::Current()),
364 task_queue_factory,
365 TaskQueueBase::Current(),
366 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200367 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200368 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200369 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200370 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200371 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200372 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200373 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200374 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100375 }
perkj803d97f2016-11-01 11:45:46 -0700376
Henrik Boström381d1092020-05-12 18:49:07 +0200377 void SetSourceAndWaitForRestrictionsUpdated(
378 rtc::VideoSourceInterface<VideoFrame>* source,
379 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200380 FakeVideoSourceRestrictionsListener listener;
381 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200382 SetSource(source, degradation_preference);
383 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200384 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200385 }
386
387 void SetSourceAndWaitForFramerateUpdated(
388 rtc::VideoSourceInterface<VideoFrame>* source,
389 const DegradationPreference& degradation_preference) {
390 overuse_detector_proxy_->framerate_updated_event()->Reset();
391 SetSource(source, degradation_preference);
392 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
393 }
394
395 void OnBitrateUpdatedAndWaitForManagedResources(
396 DataRate target_bitrate,
397 DataRate stable_target_bitrate,
398 DataRate link_allocation,
399 uint8_t fraction_lost,
400 int64_t round_trip_time_ms,
401 double cwnd_reduce_ratio) {
402 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
403 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
404 // Bitrate is updated on the encoder queue.
405 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200406 }
407
kthelgason2fc52542017-03-03 00:24:41 -0800408 // This is used as a synchronisation mechanism, to make sure that the
409 // encoder queue is not blocked before we start sending it frames.
410 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100411 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800412 }
413
Henrik Boström91aa7322020-04-28 12:24:33 +0200414 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200415 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200416 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200417 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200418 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200419 event.Set();
420 });
421 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100422 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200423 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200424
Henrik Boström91aa7322020-04-28 12:24:33 +0200425 void TriggerCpuUnderuse() {
426 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200427 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200428 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200429 event.Set();
430 });
431 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100432 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200433 }
kthelgason876222f2016-11-29 01:44:11 -0800434
Henrik Boström91aa7322020-04-28 12:24:33 +0200435 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200436 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200437 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200438 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200439 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 event.Set();
441 });
442 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100443 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200444 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200445 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200447 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200448 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 event.Set();
450 });
451 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100452 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200453 }
454
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200455 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100456 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200457 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
458 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200459 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700460};
461
Noah Richards51db4212019-06-12 06:59:12 -0700462// Simulates simulcast behavior and makes highest stream resolutions divisible
463// by 4.
464class CroppingVideoStreamFactory
465 : public VideoEncoderConfig::VideoStreamFactoryInterface {
466 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200467 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700468
469 private:
470 std::vector<VideoStream> CreateEncoderStreams(
471 int width,
472 int height,
473 const VideoEncoderConfig& encoder_config) override {
474 std::vector<VideoStream> streams = test::CreateVideoStreams(
475 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700476 return streams;
477 }
Noah Richards51db4212019-06-12 06:59:12 -0700478};
479
sprangb1ca0732017-02-01 08:38:12 -0800480class AdaptingFrameForwarder : public test::FrameForwarder {
481 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200482 explicit AdaptingFrameForwarder(TimeController* time_controller)
483 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700484 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800485
486 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200487 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800488 adaptation_enabled_ = enabled;
489 }
490
asaperssonfab67072017-04-04 05:51:49 -0700491 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200492 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800493 return adaptation_enabled_;
494 }
495
Henrik Boström1124ed12021-02-25 10:30:39 +0100496 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
497 // the resolution or frame rate was different than it is currently. If
498 // something else is modified, such as encoder resolutions, but the resolution
499 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700500 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200501 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700502 return last_wants_;
503 }
504
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200505 absl::optional<int> last_sent_width() const { return last_width_; }
506 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800507
sprangb1ca0732017-02-01 08:38:12 -0800508 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200509 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100510 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511
sprangb1ca0732017-02-01 08:38:12 -0800512 int cropped_width = 0;
513 int cropped_height = 0;
514 int out_width = 0;
515 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700516 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000517 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
518 << "w=" << video_frame.width()
519 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700520 if (adapter_.AdaptFrameResolution(
521 video_frame.width(), video_frame.height(),
522 video_frame.timestamp_us() * 1000, &cropped_width,
523 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100524 VideoFrame adapted_frame =
525 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200526 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100527 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200528 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100529 .set_timestamp_ms(99)
530 .set_rotation(kVideoRotation_0)
531 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100532 if (video_frame.has_update_rect()) {
533 adapted_frame.set_update_rect(
534 video_frame.update_rect().ScaleWithFrame(
535 video_frame.width(), video_frame.height(), 0, 0,
536 video_frame.width(), video_frame.height(), out_width,
537 out_height));
538 }
sprangc5d62e22017-04-02 23:53:04 -0700539 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800540 last_width_.emplace(adapted_frame.width());
541 last_height_.emplace(adapted_frame.height());
542 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200543 last_width_ = absl::nullopt;
544 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700545 }
sprangb1ca0732017-02-01 08:38:12 -0800546 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000547 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800548 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800549 last_width_.emplace(video_frame.width());
550 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800551 }
552 }
553
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200554 void OnOutputFormatRequest(int width, int height) {
555 absl::optional<std::pair<int, int>> target_aspect_ratio =
556 std::make_pair(width, height);
557 absl::optional<int> max_pixel_count = width * height;
558 absl::optional<int> max_fps;
559 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
560 max_fps);
561 }
562
sprangb1ca0732017-02-01 08:38:12 -0800563 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
564 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200565 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100566 rtc::VideoSinkWants prev_wants = sink_wants_locked();
567 bool did_adapt =
568 prev_wants.max_pixel_count != wants.max_pixel_count ||
569 prev_wants.target_pixel_count != wants.target_pixel_count ||
570 prev_wants.max_framerate_fps != wants.max_framerate_fps;
571 if (did_adapt) {
572 last_wants_ = prev_wants;
573 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100574 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200575 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800576 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200577
578 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800579 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200580 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
581 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200582 absl::optional<int> last_width_;
583 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800584};
sprangc5d62e22017-04-02 23:53:04 -0700585
Niels Möller213618e2018-07-24 09:29:58 +0200586// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700587class MockableSendStatisticsProxy : public SendStatisticsProxy {
588 public:
589 MockableSendStatisticsProxy(Clock* clock,
590 const VideoSendStream::Config& config,
591 VideoEncoderConfig::ContentType content_type)
592 : SendStatisticsProxy(clock, config, content_type) {}
593
594 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200595 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700596 if (mock_stats_)
597 return *mock_stats_;
598 return SendStatisticsProxy::GetStats();
599 }
600
Niels Möller213618e2018-07-24 09:29:58 +0200601 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200602 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200603 if (mock_stats_)
604 return mock_stats_->input_frame_rate;
605 return SendStatisticsProxy::GetInputFrameRate();
606 }
sprangc5d62e22017-04-02 23:53:04 -0700607 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200608 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700609 mock_stats_.emplace(stats);
610 }
611
612 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200613 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700614 mock_stats_.reset();
615 }
616
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200617 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
618 on_frame_dropped_ = std::move(callback);
619 }
620
sprangc5d62e22017-04-02 23:53:04 -0700621 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200622 void OnFrameDropped(DropReason reason) override {
623 SendStatisticsProxy::OnFrameDropped(reason);
624 if (on_frame_dropped_)
625 on_frame_dropped_(reason);
626 }
627
Markus Handella3765182020-07-08 13:13:32 +0200628 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200629 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200630 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700631};
632
Markus Handellb4e96d42021-11-05 12:00:55 +0100633class SimpleVideoStreamEncoderFactory {
634 public:
635 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
636 public:
637 using VideoStreamEncoder::VideoStreamEncoder;
638 ~AdaptedVideoStreamEncoder() { Stop(); }
639 };
640
641 SimpleVideoStreamEncoderFactory()
642 : time_controller_(Timestamp::Millis(0)),
643 task_queue_factory_(time_controller_.CreateTaskQueueFactory()),
644 stats_proxy_(std::make_unique<MockableSendStatisticsProxy>(
645 time_controller_.GetClock(),
646 VideoSendStream::Config(nullptr),
647 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
648 encoder_settings_(
649 VideoEncoder::Capabilities(/*loss_notification=*/false)),
650 fake_encoder_(time_controller_.GetClock()),
651 encoder_factory_(&fake_encoder_) {
652 encoder_settings_.encoder_factory = &encoder_factory_;
653 }
654
655 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
656 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter) {
657 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
658 time_controller_.GetClock(),
659 /*number_of_cores=*/1,
660 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
661 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
662 std::move(zero_hertz_adapter), task_queue_factory_.get(),
663 TaskQueueBase::Current(),
664 VideoStreamEncoder::BitrateAllocationCallbackType::
665 kVideoBitrateAllocation);
666 result->SetSink(&sink_, /*rotation_applied=*/false);
667 return result;
668 }
669
670 private:
671 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
672 public:
673 ~NullEncoderSink() override = default;
674 void OnEncoderConfigurationChanged(
675 std::vector<VideoStream> streams,
676 bool is_svc,
677 VideoEncoderConfig::ContentType content_type,
678 int min_transmit_bitrate_bps) override {}
679 void OnBitrateAllocationUpdated(
680 const VideoBitrateAllocation& allocation) override {}
681 void OnVideoLayersAllocationUpdated(
682 VideoLayersAllocation allocation) override {}
683 Result OnEncodedImage(
684 const EncodedImage& encoded_image,
685 const CodecSpecificInfo* codec_specific_info) override {
686 return Result(EncodedImageCallback::Result::OK);
687 }
688 };
689
690 GlobalSimulatedTimeController time_controller_;
691 std::unique_ptr<TaskQueueFactory> task_queue_factory_;
692 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
693 VideoStreamEncoderSettings encoder_settings_;
694 test::FakeEncoder fake_encoder_;
695 test::VideoEncoderProxyFactory encoder_factory_;
696 NullEncoderSink sink_;
697};
698
699class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
700 public:
701 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
702 MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override));
703 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
704};
705
philipel9b058032020-02-10 11:30:00 +0100706class MockEncoderSelector
707 : public VideoEncoderFactory::EncoderSelectorInterface {
708 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200709 MOCK_METHOD(void,
710 OnCurrentEncoder,
711 (const SdpVideoFormat& format),
712 (override));
713 MOCK_METHOD(absl::optional<SdpVideoFormat>,
714 OnAvailableBitrate,
715 (const DataRate& rate),
716 (override));
717 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100718};
719
perkj803d97f2016-11-01 11:45:46 -0700720} // namespace
721
mflodmancc3d4422017-08-03 08:27:51 -0700722class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700723 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200724 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700725
mflodmancc3d4422017-08-03 08:27:51 -0700726 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700727 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700728 codec_width_(320),
729 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200730 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200731 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200732 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700733 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200734 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700735 video_send_config_,
736 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200737 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700738
739 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700740 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700741 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200742 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800743 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200744 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200745 video_send_config_.rtp.payload_name = "FAKE";
746 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700747
Per512ecb32016-09-23 15:52:06 +0200748 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200749 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200750 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
751 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
752 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100753 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700754
Niels Möllerf1338562018-04-26 09:51:47 +0200755 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800756 }
757
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100758 void ConfigureEncoder(
759 VideoEncoderConfig video_encoder_config,
760 VideoStreamEncoder::BitrateAllocationCallbackType
761 allocation_callback_type =
762 VideoStreamEncoder::BitrateAllocationCallbackType::
763 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700764 if (video_stream_encoder_)
765 video_stream_encoder_->Stop();
766 video_stream_encoder_.reset(new VideoStreamEncoderUnderTest(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200767 &time_controller_, GetTaskQueueFactory(), stats_proxy_.get(),
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100768 video_send_config_.encoder_settings, allocation_callback_type));
Asa Persson606d3cb2021-10-04 10:07:11 +0200769 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700770 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700771 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200772 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700773 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200774 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700775 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800776 }
777
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100778 void ResetEncoder(const std::string& payload_name,
779 size_t num_streams,
780 size_t num_temporal_layers,
781 unsigned char num_spatial_layers,
782 bool screenshare,
783 VideoStreamEncoder::BitrateAllocationCallbackType
784 allocation_callback_type =
785 VideoStreamEncoder::BitrateAllocationCallbackType::
786 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200787 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800788
789 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200790 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
791 num_streams, &video_encoder_config);
792 for (auto& layer : video_encoder_config.simulcast_layers) {
793 layer.num_temporal_layers = num_temporal_layers;
794 layer.max_framerate = kDefaultFramerate;
795 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100796 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200797 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700798 video_encoder_config.content_type =
799 screenshare ? VideoEncoderConfig::ContentType::kScreen
800 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700801 if (payload_name == "VP9") {
802 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
803 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200804 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700805 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200806 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
807 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700808 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100809 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700810 }
811
sprang57c2fff2017-01-16 06:24:02 -0800812 VideoFrame CreateFrame(int64_t ntp_time_ms,
813 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200814 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200815 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200816 destruction_event, codec_width_, codec_height_))
817 .set_ntp_time_ms(ntp_time_ms)
818 .set_timestamp_ms(99)
819 .set_rotation(kVideoRotation_0)
820 .build();
perkj26091b12016-09-01 01:17:40 -0700821 }
822
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100823 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
824 rtc::Event* destruction_event,
825 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200826 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200827 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200828 destruction_event, codec_width_, codec_height_))
829 .set_ntp_time_ms(ntp_time_ms)
830 .set_timestamp_ms(99)
831 .set_rotation(kVideoRotation_0)
832 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
833 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100834 }
835
sprang57c2fff2017-01-16 06:24:02 -0800836 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200837 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
838 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200839 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200840 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200841 .set_ntp_time_ms(ntp_time_ms)
842 .set_timestamp_ms(ntp_time_ms)
843 .set_rotation(kVideoRotation_0)
844 .build();
perkj803d97f2016-11-01 11:45:46 -0700845 }
846
Evan Shrubsole895556e2020-10-05 09:15:13 +0200847 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200848 return VideoFrame::Builder()
849 .set_video_frame_buffer(NV12Buffer::Create(width, height))
850 .set_ntp_time_ms(ntp_time_ms)
851 .set_timestamp_ms(ntp_time_ms)
852 .set_rotation(kVideoRotation_0)
853 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200854 }
855
Noah Richards51db4212019-06-12 06:59:12 -0700856 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
857 rtc::Event* destruction_event,
858 int width,
859 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200860 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200861 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200862 destruction_event, width, height))
863 .set_ntp_time_ms(ntp_time_ms)
864 .set_timestamp_ms(99)
865 .set_rotation(kVideoRotation_0)
866 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700867 }
868
Evan Shrubsole895556e2020-10-05 09:15:13 +0200869 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
870 rtc::Event* destruction_event,
871 int width,
872 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200873 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200874 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200875 destruction_event, width, height))
876 .set_ntp_time_ms(ntp_time_ms)
877 .set_timestamp_ms(99)
878 .set_rotation(kVideoRotation_0)
879 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200880 }
881
Noah Richards51db4212019-06-12 06:59:12 -0700882 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
883 rtc::Event* destruction_event) const {
884 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
885 codec_height_);
886 }
887
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100888 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200889 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200890 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100891
892 video_source_.IncomingCapturedFrame(
893 CreateFrame(1, codec_width_, codec_height_));
894 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200895 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100896 }
897
sprang4847ae62017-06-27 07:06:52 -0700898 void WaitForEncodedFrame(int64_t expected_ntp_time) {
899 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200900 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700901 }
902
903 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
904 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200905 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700906 return ok;
907 }
908
909 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
910 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200911 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700912 }
913
914 void ExpectDroppedFrame() {
915 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200916 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700917 }
918
919 bool WaitForFrame(int64_t timeout_ms) {
920 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200921 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700922 return ok;
923 }
924
perkj26091b12016-09-01 01:17:40 -0700925 class TestEncoder : public test::FakeEncoder {
926 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200927 explicit TestEncoder(TimeController* time_controller)
928 : FakeEncoder(time_controller->GetClock()),
929 time_controller_(time_controller) {
930 RTC_DCHECK(time_controller_);
931 }
perkj26091b12016-09-01 01:17:40 -0700932
perkjfa10b552016-10-02 23:45:26 -0700933 void BlockNextEncode() {
Markus Handella3765182020-07-08 13:13:32 +0200934 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700935 block_next_encode_ = true;
936 }
937
Erik Språngaed30702018-11-05 12:57:17 +0100938 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200939 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200940 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100941 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100942 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100943 info.scaling_settings = VideoEncoder::ScalingSettings(
944 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100945 }
946 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100947 for (int i = 0; i < kMaxSpatialLayers; ++i) {
948 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100949 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100950 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100951 for (int tid = 0; tid < num_layers; ++tid)
952 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100953 }
954 }
Erik Språngaed30702018-11-05 12:57:17 +0100955 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200956
957 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100958 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200959 info.apply_alignment_to_all_simulcast_layers =
960 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200961 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800962 if (is_qp_trusted_.has_value()) {
963 info.is_qp_trusted = is_qp_trusted_;
964 }
Erik Språngaed30702018-11-05 12:57:17 +0100965 return info;
kthelgason876222f2016-11-29 01:44:11 -0800966 }
967
Erik Språngb7cb7b52019-02-26 15:52:33 +0100968 int32_t RegisterEncodeCompleteCallback(
969 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200970 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100971 encoded_image_callback_ = callback;
972 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
973 }
974
perkjfa10b552016-10-02 23:45:26 -0700975 void ContinueEncode() { continue_encode_event_.Set(); }
976
977 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
978 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200979 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700980 EXPECT_EQ(timestamp_, timestamp);
981 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
982 }
983
kthelgason2fc52542017-03-03 00:24:41 -0800984 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +0200985 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -0800986 quality_scaling_ = b;
987 }
kthelgasonad9010c2017-02-14 00:46:51 -0800988
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100989 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +0200990 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100991 requested_resolution_alignment_ = requested_resolution_alignment;
992 }
993
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200994 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
995 MutexLock lock(&local_mutex_);
996 apply_alignment_to_all_simulcast_layers_ = b;
997 }
998
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100999 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001000 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001001 is_hardware_accelerated_ = is_hardware_accelerated;
1002 }
1003
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001004 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1005 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001006 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001007 temporal_layers_supported_[spatial_idx] = supported;
1008 }
1009
Sergey Silkin6456e352019-07-08 17:56:40 +02001010 void SetResolutionBitrateLimits(
1011 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001012 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001013 resolution_bitrate_limits_ = thresholds;
1014 }
1015
sprangfe627f32017-03-29 08:24:59 -07001016 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001017 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001018 force_init_encode_failed_ = force_failure;
1019 }
1020
Niels Möller6bb5ab92019-01-11 11:11:10 +01001021 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001022 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001023 rate_factor_ = rate_factor;
1024 }
1025
Erik Språngd7329ca2019-02-21 21:19:53 +01001026 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001027 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001028 return last_framerate_;
1029 }
1030
Erik Språngd7329ca2019-02-21 21:19:53 +01001031 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001032 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001033 return last_update_rect_;
1034 }
1035
Niels Möller87e2d782019-03-07 10:18:23 +01001036 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001037 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001038 return last_frame_types_;
1039 }
1040
1041 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001042 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001043 keyframe ? VideoFrameType::kVideoFrameKey
1044 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001045 {
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001047 last_frame_types_ = frame_type;
1048 }
Niels Möllerb859b322019-03-07 12:40:01 +01001049 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001050 }
1051
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001052 void InjectEncodedImage(const EncodedImage& image,
1053 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001054 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001055 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001056 }
1057
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001058 void SetEncodedImageData(
1059 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001060 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001061 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001062 }
1063
Erik Språngd7329ca2019-02-21 21:19:53 +01001064 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001065 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001066 expect_null_frame_ = true;
1067 }
1068
Erik Språng5056af02019-09-02 15:53:11 +02001069 absl::optional<VideoEncoder::RateControlParameters>
1070 GetAndResetLastRateControlSettings() {
1071 auto settings = last_rate_control_settings_;
1072 last_rate_control_settings_.reset();
1073 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001074 }
1075
Henrik Boström56db9ff2021-03-24 09:06:45 +01001076 int GetLastInputWidth() const {
1077 MutexLock lock(&local_mutex_);
1078 return last_input_width_;
1079 }
1080
1081 int GetLastInputHeight() const {
1082 MutexLock lock(&local_mutex_);
1083 return last_input_height_;
1084 }
1085
Evan Shrubsole895556e2020-10-05 09:15:13 +02001086 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1087 MutexLock lock(&local_mutex_);
1088 return last_input_pixel_format_;
1089 }
1090
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001091 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001093 return num_set_rates_;
1094 }
1095
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001096 void SetPreferredPixelFormats(
1097 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1098 pixel_formats) {
1099 MutexLock lock(&local_mutex_);
1100 preferred_pixel_formats_ = std::move(pixel_formats);
1101 }
1102
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001103 void SetIsQpTrusted(absl::optional<bool> trusted) {
1104 MutexLock lock(&local_mutex_);
1105 is_qp_trusted_ = trusted;
1106 }
1107
perkjfa10b552016-10-02 23:45:26 -07001108 private:
perkj26091b12016-09-01 01:17:40 -07001109 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001110 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001111 bool block_encode;
1112 {
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001114 if (expect_null_frame_) {
1115 EXPECT_EQ(input_image.timestamp(), 0u);
1116 EXPECT_EQ(input_image.width(), 1);
1117 last_frame_types_ = *frame_types;
1118 expect_null_frame_ = false;
1119 } else {
1120 EXPECT_GT(input_image.timestamp(), timestamp_);
1121 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1122 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1123 }
perkj26091b12016-09-01 01:17:40 -07001124
1125 timestamp_ = input_image.timestamp();
1126 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001127 last_input_width_ = input_image.width();
1128 last_input_height_ = input_image.height();
perkj26091b12016-09-01 01:17:40 -07001129 block_encode = block_next_encode_;
1130 block_next_encode_ = false;
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001131 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001132 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001133 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001134 }
Niels Möllerb859b322019-03-07 12:40:01 +01001135 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001136 if (block_encode)
perkja49cbd32016-09-16 07:53:41 -07001137 EXPECT_TRUE(continue_encode_event_.Wait(kDefaultTimeoutMs));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001138
perkj26091b12016-09-01 01:17:40 -07001139 return result;
1140 }
1141
Niels Möller08ae7ce2020-09-23 15:58:12 +02001142 CodecSpecificInfo EncodeHook(
1143 EncodedImage& encoded_image,
1144 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001145 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001146 {
1147 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001148 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001149 }
1150 MutexLock lock(&local_mutex_);
1151 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001152 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001153 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001154 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001155 }
1156
sprangfe627f32017-03-29 08:24:59 -07001157 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001158 const Settings& settings) override {
1159 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001160
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001162 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001163
Erik Språng82fad3d2018-03-21 09:57:23 +01001164 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001165 // Simulate setting up temporal layers, in order to validate the life
1166 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001167 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001168 frame_buffer_controller_ =
1169 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001170 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001171 if (force_init_encode_failed_) {
1172 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001173 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001174 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001175
Erik Språngb7cb7b52019-02-26 15:52:33 +01001176 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001177 return res;
1178 }
1179
Erik Språngb7cb7b52019-02-26 15:52:33 +01001180 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001181 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001182 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1183 initialized_ = EncoderState::kUninitialized;
1184 return FakeEncoder::Release();
1185 }
1186
Erik Språng16cb8f52019-04-12 13:59:09 +02001187 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001188 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001189 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001190 VideoBitrateAllocation adjusted_rate_allocation;
1191 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1192 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001193 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001194 adjusted_rate_allocation.SetBitrate(
1195 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001196 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001197 rate_factor_));
1198 }
1199 }
1200 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001201 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001202 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001203 RateControlParameters adjusted_paramters = parameters;
1204 adjusted_paramters.bitrate = adjusted_rate_allocation;
1205 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001206 }
1207
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001208 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001209 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001210 enum class EncoderState {
1211 kUninitialized,
1212 kInitializationFailed,
1213 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001214 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
1215 bool block_next_encode_ RTC_GUARDED_BY(local_mutex_) = false;
perkj26091b12016-09-01 01:17:40 -07001216 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001217 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1218 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1219 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1220 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1221 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1222 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001223 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1224 false;
Markus Handella3765182020-07-08 13:13:32 +02001225 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001226 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1227 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001228 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001229 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001230 absl::optional<bool>
1231 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001232 local_mutex_);
1233 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1234 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1235 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001236 absl::optional<VideoEncoder::RateControlParameters>
1237 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001238 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1239 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001240 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001241 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001242 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1243 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001244 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001245 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001246 RTC_GUARDED_BY(local_mutex_);
1247 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001248 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1249 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001250 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1251 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001252 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001253 };
1254
mflodmancc3d4422017-08-03 08:27:51 -07001255 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001256 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001257 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1258 : time_controller_(time_controller), test_encoder_(test_encoder) {
1259 RTC_DCHECK(time_controller_);
1260 }
perkj26091b12016-09-01 01:17:40 -07001261
perkj26091b12016-09-01 01:17:40 -07001262 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001263 EXPECT_TRUE(
1264 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1265 }
1266
1267 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1268 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001269 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001270 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001271 return false;
perkj26091b12016-09-01 01:17:40 -07001272 {
Markus Handella3765182020-07-08 13:13:32 +02001273 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001274 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001275 }
1276 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001277 return true;
perkj26091b12016-09-01 01:17:40 -07001278 }
1279
sprangb1ca0732017-02-01 08:38:12 -08001280 void WaitForEncodedFrame(uint32_t expected_width,
1281 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001282 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001283 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001284 }
1285
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001286 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001287 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001288 uint32_t width = 0;
1289 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001290 {
Markus Handella3765182020-07-08 13:13:32 +02001291 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001292 width = last_width_;
1293 height = last_height_;
1294 }
1295 EXPECT_EQ(expected_height, height);
1296 EXPECT_EQ(expected_width, width);
1297 }
1298
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001299 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1300 VideoRotation rotation;
1301 {
Markus Handella3765182020-07-08 13:13:32 +02001302 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001303 rotation = last_rotation_;
1304 }
1305 EXPECT_EQ(expected_rotation, rotation);
1306 }
1307
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001308 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001309
sprangc5d62e22017-04-02 23:53:04 -07001310 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001311 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001312 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001313 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001314 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001315 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001316 }
1317
perkj26091b12016-09-01 01:17:40 -07001318 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001319 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001320 expect_frames_ = false;
1321 }
1322
asaperssonfab67072017-04-04 05:51:49 -07001323 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001324 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001325 return number_of_reconfigurations_;
1326 }
1327
asaperssonfab67072017-04-04 05:51:49 -07001328 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001329 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001330 return min_transmit_bitrate_bps_;
1331 }
1332
Erik Språngd7329ca2019-02-21 21:19:53 +01001333 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001334 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001335 num_expected_layers_ = num_layers;
1336 }
1337
Erik Språngb7cb7b52019-02-26 15:52:33 +01001338 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001339 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001340 return last_capture_time_ms_;
1341 }
1342
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001343 const EncodedImage& GetLastEncodedImage() {
1344 MutexLock lock(&mutex_);
1345 return last_encoded_image_;
1346 }
1347
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001348 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001349 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001350 return std::move(last_encoded_image_data_);
1351 }
1352
Per Kjellanderdcef6412020-10-07 15:09:05 +02001353 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1354 MutexLock lock(&mutex_);
1355 return last_bitrate_allocation_;
1356 }
1357
1358 int number_of_bitrate_allocations() const {
1359 MutexLock lock(&mutex_);
1360 return number_of_bitrate_allocations_;
1361 }
1362
Per Kjellandera9434842020-10-15 17:53:22 +02001363 VideoLayersAllocation GetLastVideoLayersAllocation() {
1364 MutexLock lock(&mutex_);
1365 return last_layers_allocation_;
1366 }
1367
1368 int number_of_layers_allocations() const {
1369 MutexLock lock(&mutex_);
1370 return number_of_layers_allocations_;
1371 }
1372
perkj26091b12016-09-01 01:17:40 -07001373 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001374 Result OnEncodedImage(
1375 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001376 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001377 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001378 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001379 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001380 last_encoded_image_data_ = std::vector<uint8_t>(
1381 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001382 uint32_t timestamp = encoded_image.Timestamp();
1383 if (last_timestamp_ != timestamp) {
1384 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001385 last_width_ = encoded_image._encodedWidth;
1386 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001387 } else {
1388 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001389 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1390 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001391 }
1392 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001393 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001394 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001395 if (num_received_layers_ == num_expected_layers_) {
1396 encoded_frame_event_.Set();
1397 }
sprangb1ca0732017-02-01 08:38:12 -08001398 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001399 }
1400
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001401 void OnEncoderConfigurationChanged(
1402 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001403 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001404 VideoEncoderConfig::ContentType content_type,
1405 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001406 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001407 ++number_of_reconfigurations_;
1408 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1409 }
1410
Per Kjellanderdcef6412020-10-07 15:09:05 +02001411 void OnBitrateAllocationUpdated(
1412 const VideoBitrateAllocation& allocation) override {
1413 MutexLock lock(&mutex_);
1414 ++number_of_bitrate_allocations_;
1415 last_bitrate_allocation_ = allocation;
1416 }
1417
Per Kjellandera9434842020-10-15 17:53:22 +02001418 void OnVideoLayersAllocationUpdated(
1419 VideoLayersAllocation allocation) override {
1420 MutexLock lock(&mutex_);
1421 ++number_of_layers_allocations_;
1422 last_layers_allocation_ = allocation;
1423 rtc::StringBuilder log;
1424 for (const auto& layer : allocation.active_spatial_layers) {
1425 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1426 << "[";
1427 for (const auto target_bitrate :
1428 layer.target_bitrate_per_temporal_layer) {
1429 log << target_bitrate.kbps() << ",";
1430 }
1431 log << "]";
1432 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001433 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001434 }
1435
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001436 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001437 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001438 TestEncoder* test_encoder_;
1439 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001440 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001441 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001442 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001443 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001444 uint32_t last_height_ = 0;
1445 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001446 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001447 size_t num_expected_layers_ = 1;
1448 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001449 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001450 int number_of_reconfigurations_ = 0;
1451 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001452 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1453 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001454 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1455 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001456 };
1457
Sergey Silkin5ee69672019-07-02 14:18:34 +02001458 class VideoBitrateAllocatorProxyFactory
1459 : public VideoBitrateAllocatorFactory {
1460 public:
1461 VideoBitrateAllocatorProxyFactory()
1462 : bitrate_allocator_factory_(
1463 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1464
1465 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1466 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001467 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001468 codec_config_ = codec;
1469 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1470 }
1471
1472 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001473 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001474 return codec_config_;
1475 }
1476
1477 private:
1478 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1479
Markus Handella3765182020-07-08 13:13:32 +02001480 mutable Mutex mutex_;
1481 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001482 };
1483
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001484 Clock* clock() { return time_controller_.GetClock(); }
1485 void AdvanceTime(TimeDelta duration) {
1486 time_controller_.AdvanceTime(duration);
1487 }
1488
1489 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1490
1491 protected:
1492 virtual TaskQueueFactory* GetTaskQueueFactory() {
1493 return time_controller_.GetTaskQueueFactory();
1494 }
1495
1496 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001497 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001498 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001499 int codec_width_;
1500 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001501 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001502 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001503 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001504 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001505 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001506 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001507 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001508 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001509};
1510
mflodmancc3d4422017-08-03 08:27:51 -07001511TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001514 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001515 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001516 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001517 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001518 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001519}
1520
mflodmancc3d4422017-08-03 08:27:51 -07001521TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001522 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001523 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001524 // The encoder will cache up to one frame for a short duration. Adding two
1525 // frames means that the first frame will be dropped and the second frame will
1526 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001527 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001528 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001529 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001530 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001531 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001532
Henrik Boström381d1092020-05-12 18:49:07 +02001533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001534 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001535
Sebastian Janssona3177052018-04-10 13:05:49 +02001536 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001537 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001538 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1539
1540 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001541 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001542}
1543
mflodmancc3d4422017-08-03 08:27:51 -07001544TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001546 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001547 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001548 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001549
Henrik Boström381d1092020-05-12 18:49:07 +02001550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001551 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1552
Sebastian Janssona3177052018-04-10 13:05:49 +02001553 // The encoder will cache up to one frame for a short duration. Adding two
1554 // frames means that the first frame will be dropped and the second frame will
1555 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001556 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001557 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001558
Henrik Boström381d1092020-05-12 18:49:07 +02001559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001560 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001561 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001562 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1563 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001564 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001565}
1566
mflodmancc3d4422017-08-03 08:27:51 -07001567TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001569 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001570 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001571 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001572
1573 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001574 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001575
perkja49cbd32016-09-16 07:53:41 -07001576 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001577 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001578 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001579}
1580
mflodmancc3d4422017-08-03 08:27:51 -07001581TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001583 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001584
perkja49cbd32016-09-16 07:53:41 -07001585 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001586 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001587
mflodmancc3d4422017-08-03 08:27:51 -07001588 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001589 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001590 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001591 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1592 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001593}
1594
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001595class VideoStreamEncoderBlockedTest : public VideoStreamEncoderTest {
1596 public:
1597 VideoStreamEncoderBlockedTest() {}
1598
1599 TaskQueueFactory* GetTaskQueueFactory() override {
1600 return task_queue_factory_.get();
1601 }
1602
1603 private:
1604 std::unique_ptr<TaskQueueFactory> task_queue_factory_ =
1605 CreateDefaultTaskQueueFactory();
1606};
1607
1608TEST_F(VideoStreamEncoderBlockedTest, DropsPendingFramesOnSlowEncode) {
Henrik Boström381d1092020-05-12 18:49:07 +02001609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001610 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001611
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001612 int dropped_count = 0;
1613 stats_proxy_->SetDroppedFrameCallback(
1614 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1615 ++dropped_count;
1616 });
1617
perkj26091b12016-09-01 01:17:40 -07001618 fake_encoder_.BlockNextEncode();
perkja49cbd32016-09-16 07:53:41 -07001619 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001620 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001621 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
1622 // call to ContinueEncode.
perkja49cbd32016-09-16 07:53:41 -07001623 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1624 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001625 fake_encoder_.ContinueEncode();
sprang4847ae62017-06-27 07:06:52 -07001626 WaitForEncodedFrame(3);
perkj26091b12016-09-01 01:17:40 -07001627
mflodmancc3d4422017-08-03 08:27:51 -07001628 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001629
1630 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001631}
1632
Henrik Boström56db9ff2021-03-24 09:06:45 +01001633TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001635 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001636
1637 rtc::Event frame_destroyed_event;
1638 video_source_.IncomingCapturedFrame(
1639 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001640 WaitForEncodedFrame(1);
1641 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1642 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001643 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1644 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001645 video_stream_encoder_->Stop();
1646}
1647
Henrik Boström56db9ff2021-03-24 09:06:45 +01001648TEST_F(VideoStreamEncoderTest,
1649 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001650 // Use the cropping factory.
1651 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001652 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001653 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1654 kMaxPayloadLength);
1655 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1656
1657 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001659 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001660 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1661 WaitForEncodedFrame(1);
1662 // The encoder will have been configured once.
1663 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001664 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1665 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001666
1667 // Now send in a fake frame that needs to be cropped as the width/height
1668 // aren't divisible by 4 (see CreateEncoderStreams above).
1669 rtc::Event frame_destroyed_event;
1670 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1671 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001672 WaitForEncodedFrame(2);
1673 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1674 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001675 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1676 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001677 video_stream_encoder_->Stop();
1678}
1679
Evan Shrubsole895556e2020-10-05 09:15:13 +02001680TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001682 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001683
1684 video_source_.IncomingCapturedFrame(
1685 CreateNV12Frame(1, codec_width_, codec_height_));
1686 WaitForEncodedFrame(1);
1687 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1688 fake_encoder_.GetLastInputPixelFormat());
1689 video_stream_encoder_->Stop();
1690}
1691
Henrik Boström56db9ff2021-03-24 09:06:45 +01001692TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001694 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001695
1696 fake_encoder_.SetPreferredPixelFormats({});
1697
1698 rtc::Event frame_destroyed_event;
1699 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1700 1, &frame_destroyed_event, codec_width_, codec_height_));
1701 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001702 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001703 fake_encoder_.GetLastInputPixelFormat());
1704 video_stream_encoder_->Stop();
1705}
1706
Henrik Boström56db9ff2021-03-24 09:06:45 +01001707TEST_F(VideoStreamEncoderTest,
1708 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001710 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001711
1712 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1713
1714 rtc::Event frame_destroyed_event;
1715 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1716 1, &frame_destroyed_event, codec_width_, codec_height_));
1717 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001718 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001719 fake_encoder_.GetLastInputPixelFormat());
1720 video_stream_encoder_->Stop();
1721}
1722
Henrik Boström56db9ff2021-03-24 09:06:45 +01001723TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001725 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001726
1727 // Fake NV12 native frame does not allow mapping to I444.
1728 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1729
1730 rtc::Event frame_destroyed_event;
1731 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1732 1, &frame_destroyed_event, codec_width_, codec_height_));
1733 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001734 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001735 fake_encoder_.GetLastInputPixelFormat());
1736 video_stream_encoder_->Stop();
1737}
1738
Henrik Boström56db9ff2021-03-24 09:06:45 +01001739TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001741 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001742
1743 rtc::Event frame_destroyed_event;
1744 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1745 1, &frame_destroyed_event, codec_width_, codec_height_));
1746 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001747 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001748 fake_encoder_.GetLastInputPixelFormat());
1749 video_stream_encoder_->Stop();
1750}
1751
Ying Wang9b881ab2020-02-07 14:29:32 +01001752TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001755 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1756 WaitForEncodedFrame(1);
1757
Henrik Boström381d1092020-05-12 18:49:07 +02001758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001759 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001760 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1761 // frames. Adding two frames means that the first frame will be dropped and
1762 // the second frame will be sent to the encoder.
1763 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1764 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1765 WaitForEncodedFrame(3);
1766 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1767 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1768 WaitForEncodedFrame(5);
1769 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1770 video_stream_encoder_->Stop();
1771}
1772
mflodmancc3d4422017-08-03 08:27:51 -07001773TEST_F(VideoStreamEncoderTest,
1774 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001775 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001776 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001777 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001778
1779 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001780 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001781 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001782 // The encoder will have been configured once when the first frame is
1783 // received.
1784 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001785
1786 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001787 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001788 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001789 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001790 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001791
1792 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001793 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001794 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001795 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001796 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001797
mflodmancc3d4422017-08-03 08:27:51 -07001798 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001799}
1800
mflodmancc3d4422017-08-03 08:27:51 -07001801TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001803 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001804
1805 // Capture a frame and wait for it to synchronize with the encoder thread.
1806 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001807 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001808 // The encoder will have been configured once.
1809 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001810 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1811 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001812
1813 codec_width_ *= 2;
1814 codec_height_ *= 2;
1815 // Capture a frame with a higher resolution and wait for it to synchronize
1816 // with the encoder thread.
1817 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001818 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001819 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1820 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001821 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001822
mflodmancc3d4422017-08-03 08:27:51 -07001823 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001824}
1825
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001826TEST_F(VideoStreamEncoderTest,
1827 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001830
1831 // Capture a frame and wait for it to synchronize with the encoder thread.
1832 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1833 WaitForEncodedFrame(1);
1834
1835 VideoEncoderConfig video_encoder_config;
1836 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1837 // Changing the max payload data length recreates encoder.
1838 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1839 kMaxPayloadLength / 2);
1840
1841 // Capture a frame and wait for it to synchronize with the encoder thread.
1842 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1843 WaitForEncodedFrame(2);
1844 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1845
1846 video_stream_encoder_->Stop();
1847}
1848
Sergey Silkin5ee69672019-07-02 14:18:34 +02001849TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001851 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001852
1853 VideoEncoderConfig video_encoder_config;
1854 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001855 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1856 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001857 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1858 kMaxPayloadLength);
1859
1860 // Capture a frame and wait for it to synchronize with the encoder thread.
1861 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1862 WaitForEncodedFrame(1);
1863 // The encoder will have been configured once when the first frame is
1864 // received.
1865 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001866 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001867 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001868 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001869 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1870
Sergey Silkin6456e352019-07-08 17:56:40 +02001871 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1872 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001873 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1874 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001875 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1876 kMaxPayloadLength);
1877
1878 // Capture a frame and wait for it to synchronize with the encoder thread.
1879 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1880 WaitForEncodedFrame(2);
1881 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1882 // Bitrate limits have changed - rate allocator should be reconfigured,
1883 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001884 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001885 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001886 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001887 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001888 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001889
1890 video_stream_encoder_->Stop();
1891}
1892
Sergey Silkin6456e352019-07-08 17:56:40 +02001893TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001894 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001897
Sergey Silkincd02eba2020-01-20 14:48:40 +01001898 const uint32_t kMinEncBitrateKbps = 100;
1899 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001900 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001901 /*frame_size_pixels=*/codec_width_ * codec_height_,
1902 /*min_start_bitrate_bps=*/0,
1903 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1904 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001905 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1906
Sergey Silkincd02eba2020-01-20 14:48:40 +01001907 VideoEncoderConfig video_encoder_config;
1908 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1909 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1910 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1911 (kMinEncBitrateKbps + 1) * 1000;
1912 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1913 kMaxPayloadLength);
1914
1915 // When both encoder and app provide bitrate limits, the intersection of
1916 // provided sets should be used.
1917 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1918 WaitForEncodedFrame(1);
1919 EXPECT_EQ(kMaxEncBitrateKbps,
1920 bitrate_allocator_factory_.codec_config().maxBitrate);
1921 EXPECT_EQ(kMinEncBitrateKbps + 1,
1922 bitrate_allocator_factory_.codec_config().minBitrate);
1923
1924 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1925 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1926 (kMinEncBitrateKbps - 1) * 1000;
1927 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1928 kMaxPayloadLength);
1929 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001930 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001931 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001932 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001933 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001934 bitrate_allocator_factory_.codec_config().minBitrate);
1935
Sergey Silkincd02eba2020-01-20 14:48:40 +01001936 video_stream_encoder_->Stop();
1937}
1938
1939TEST_F(VideoStreamEncoderTest,
1940 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001942 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001943
1944 const uint32_t kMinAppBitrateKbps = 100;
1945 const uint32_t kMaxAppBitrateKbps = 200;
1946 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1947 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1948 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1949 /*frame_size_pixels=*/codec_width_ * codec_height_,
1950 /*min_start_bitrate_bps=*/0,
1951 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1952 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1953 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1954
1955 VideoEncoderConfig video_encoder_config;
1956 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1957 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1958 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1959 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001960 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1961 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962
Sergey Silkincd02eba2020-01-20 14:48:40 +01001963 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1964 WaitForEncodedFrame(1);
1965 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001966 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001967 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001968 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001969
1970 video_stream_encoder_->Stop();
1971}
1972
1973TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001974 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001976 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001977
1978 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001979 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001980 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001981 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001982 fake_encoder_.SetResolutionBitrateLimits(
1983 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1984
1985 VideoEncoderConfig video_encoder_config;
1986 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1987 video_encoder_config.max_bitrate_bps = 0;
1988 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1989 kMaxPayloadLength);
1990
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001992 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1993 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001994 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1995 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001996 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1997 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1998
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001999 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002000 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2001 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002002 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2003 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002004 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2006
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002007 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002008 // encoder for 360p should be used.
2009 video_source_.IncomingCapturedFrame(
2010 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2011 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002012 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2013 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002014 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2015 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2016
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002018 // ignored.
2019 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2020 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002021 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2022 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002023 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2024 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2026 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002027 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2028 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2029
2030 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2031 // for 270p should be used.
2032 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2033 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002034 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2035 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002036 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2037 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2038
2039 video_stream_encoder_->Stop();
2040}
2041
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002042TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002044 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002045
2046 VideoEncoderConfig video_encoder_config;
2047 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2048 video_encoder_config.max_bitrate_bps = 0;
2049 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2050 kMaxPayloadLength);
2051
2052 // Encode 720p frame to get the default encoder target bitrate.
2053 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2054 WaitForEncodedFrame(1);
2055 const uint32_t kDefaultTargetBitrateFor720pKbps =
2056 bitrate_allocator_factory_.codec_config()
2057 .simulcastStream[0]
2058 .targetBitrate;
2059
2060 // Set the max recommended encoder bitrate to something lower than the default
2061 // target bitrate.
2062 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2063 1280 * 720, 10 * 1000, 10 * 1000,
2064 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2065 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2066
2067 // Change resolution to trigger encoder reinitialization.
2068 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2069 WaitForEncodedFrame(2);
2070 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2071 WaitForEncodedFrame(3);
2072
2073 // Ensure the target bitrate is capped by the max bitrate.
2074 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2075 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2076 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2077 .simulcastStream[0]
2078 .targetBitrate *
2079 1000,
2080 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2081
2082 video_stream_encoder_->Stop();
2083}
2084
Åsa Perssona7e34d32021-01-20 15:36:13 +01002085TEST_F(VideoStreamEncoderTest,
2086 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2087 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2088 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2089 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2090 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2091 fake_encoder_.SetResolutionBitrateLimits(
2092 {kEncoderLimits270p, kEncoderLimits360p});
2093
2094 // Two streams, highest stream active.
2095 VideoEncoderConfig config;
2096 const int kNumStreams = 2;
2097 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2098 config.max_bitrate_bps = 0;
2099 config.simulcast_layers[0].active = false;
2100 config.simulcast_layers[1].active = true;
2101 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002102 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002103 "VP8", /*max qp*/ 56, /*screencast*/ false,
2104 /*screenshare enabled*/ false);
2105 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2106
2107 // The encoder bitrate limits for 270p should be used.
2108 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2109 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002110 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002111 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002112 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002113 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002114 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002115
2116 // The encoder bitrate limits for 360p should be used.
2117 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2118 EXPECT_FALSE(WaitForFrame(1000));
2119 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002120 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002121 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002122 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002123
2124 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2125 video_source_.IncomingCapturedFrame(
2126 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2127 EXPECT_FALSE(WaitForFrame(1000));
2128 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002129 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002130 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002131 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002132
2133 // Resolution higher than 360p. Encoder limits should be ignored.
2134 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2135 EXPECT_FALSE(WaitForFrame(1000));
2136 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002137 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002138 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002139 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002140 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002141 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002142 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002143 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002144
2145 // Resolution lower than 270p. The encoder limits for 270p should be used.
2146 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2147 EXPECT_FALSE(WaitForFrame(1000));
2148 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002149 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002150 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002151 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002152
2153 video_stream_encoder_->Stop();
2154}
2155
2156TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002157 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2158 // Two streams, highest stream active.
2159 VideoEncoderConfig config;
2160 const int kNumStreams = 2;
2161 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2162 config.max_bitrate_bps = 0;
2163 config.simulcast_layers[0].active = false;
2164 config.simulcast_layers[1].active = true;
2165 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002166 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002167 "VP8", /*max qp*/ 56, /*screencast*/ false,
2168 /*screenshare enabled*/ false);
2169 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2170
2171 // Default bitrate limits for 270p should be used.
2172 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2173 kDefaultLimits270p =
2174 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002175 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002176 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2177 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002178 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002179 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002180 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002181 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002182 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002183
2184 // Default bitrate limits for 360p should be used.
2185 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2186 kDefaultLimits360p =
2187 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002188 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002189 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2190 EXPECT_FALSE(WaitForFrame(1000));
2191 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002192 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002193 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002194 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002195
2196 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2197 video_source_.IncomingCapturedFrame(
2198 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2199 EXPECT_FALSE(WaitForFrame(1000));
2200 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002201 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002202 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002204
2205 // Default bitrate limits for 540p should be used.
2206 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2207 kDefaultLimits540p =
2208 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002209 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002210 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2211 EXPECT_FALSE(WaitForFrame(1000));
2212 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002214 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002215 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002216
2217 video_stream_encoder_->Stop();
2218}
2219
2220TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002221 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2222 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2223 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2224 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2225 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2226 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2227 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2228 fake_encoder_.SetResolutionBitrateLimits(
2229 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2230
2231 // Three streams, middle stream active.
2232 VideoEncoderConfig config;
2233 const int kNumStreams = 3;
2234 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2235 config.simulcast_layers[0].active = false;
2236 config.simulcast_layers[1].active = true;
2237 config.simulcast_layers[2].active = false;
2238 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002239 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240 "VP8", /*max qp*/ 56, /*screencast*/ false,
2241 /*screenshare enabled*/ false);
2242 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2243
2244 // The encoder bitrate limits for 360p should be used.
2245 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2246 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002248 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002249 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002250 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002251 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002252
2253 // The encoder bitrate limits for 270p should be used.
2254 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2255 EXPECT_FALSE(WaitForFrame(1000));
2256 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002257 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002258 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002259 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002260
2261 video_stream_encoder_->Stop();
2262}
2263
2264TEST_F(VideoStreamEncoderTest,
2265 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2266 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2267 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2268 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2269 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2270 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2271 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2272 fake_encoder_.SetResolutionBitrateLimits(
2273 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2274
2275 // Three streams, lowest stream active.
2276 VideoEncoderConfig config;
2277 const int kNumStreams = 3;
2278 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2279 config.simulcast_layers[0].active = true;
2280 config.simulcast_layers[1].active = false;
2281 config.simulcast_layers[2].active = false;
2282 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002283 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002284 "VP8", /*max qp*/ 56, /*screencast*/ false,
2285 /*screenshare enabled*/ false);
2286 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2287
2288 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2289 // on lowest stream, limits for 270p should not be used
2290 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2291 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002292 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002293 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002295 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002296 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002297
2298 video_stream_encoder_->Stop();
2299}
2300
2301TEST_F(VideoStreamEncoderTest,
2302 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2303 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2304 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2305 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2306 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2307 fake_encoder_.SetResolutionBitrateLimits(
2308 {kEncoderLimits270p, kEncoderLimits360p});
2309 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2310
2311 // Two streams, highest stream active.
2312 VideoEncoderConfig config;
2313 const int kNumStreams = 2;
2314 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2315 config.simulcast_layers[0].active = false;
2316 config.simulcast_layers[1].active = true;
2317 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2318 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002319 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002320 "VP8", /*max qp*/ 56, /*screencast*/ false,
2321 /*screenshare enabled*/ false);
2322 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2323
2324 // The encoder bitrate limits for 270p should be used.
2325 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2326 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002327 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002328 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002329 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002330 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002331 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002332
2333 // The max configured bitrate is less than the encoder limit for 360p.
2334 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2335 EXPECT_FALSE(WaitForFrame(1000));
2336 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002337 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002338 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002339 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002340
2341 video_stream_encoder_->Stop();
2342}
2343
mflodmancc3d4422017-08-03 08:27:51 -07002344TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002345 EXPECT_TRUE(video_source_.has_sinks());
2346 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002347 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002348 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002349 EXPECT_FALSE(video_source_.has_sinks());
2350 EXPECT_TRUE(new_video_source.has_sinks());
2351
mflodmancc3d4422017-08-03 08:27:51 -07002352 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002353}
2354
mflodmancc3d4422017-08-03 08:27:51 -07002355TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002356 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002357 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002358 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002359 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002360}
2361
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002362class ResolutionAlignmentTest
2363 : public VideoStreamEncoderTest,
2364 public ::testing::WithParamInterface<
2365 ::testing::tuple<int, std::vector<double>>> {
2366 public:
2367 ResolutionAlignmentTest()
2368 : requested_alignment_(::testing::get<0>(GetParam())),
2369 scale_factors_(::testing::get<1>(GetParam())) {}
2370
2371 protected:
2372 const int requested_alignment_;
2373 const std::vector<double> scale_factors_;
2374};
2375
2376INSTANTIATE_TEST_SUITE_P(
2377 AlignmentAndScaleFactors,
2378 ResolutionAlignmentTest,
2379 ::testing::Combine(
2380 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2381 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2382 std::vector<double>{-1.0, -1.0},
2383 std::vector<double>{-1.0, -1.0, -1.0},
2384 std::vector<double>{4.0, 2.0, 1.0},
2385 std::vector<double>{9999.0, -1.0, 1.0},
2386 std::vector<double>{3.99, 2.01, 1.0},
2387 std::vector<double>{4.9, 1.7, 1.25},
2388 std::vector<double>{10.0, 4.0, 3.0},
2389 std::vector<double>{1.75, 3.5},
2390 std::vector<double>{1.5, 2.5},
2391 std::vector<double>{1.3, 1.0})));
2392
2393TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2394 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002395 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002396 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2397 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2398
2399 // Fill config with the scaling factor by which to reduce encoding size.
2400 const int num_streams = scale_factors_.size();
2401 VideoEncoderConfig config;
2402 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2403 for (int i = 0; i < num_streams; ++i) {
2404 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2405 }
2406 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002407 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002408 "VP8", /*max qp*/ 56, /*screencast*/ false,
2409 /*screenshare enabled*/ false);
2410 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2411
Henrik Boström381d1092020-05-12 18:49:07 +02002412 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002413 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2414 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002415 // Wait for all layers before triggering event.
2416 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002417
2418 // On the 1st frame, we should have initialized the encoder and
2419 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002420 int64_t timestamp_ms = kFrameIntervalMs;
2421 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2422 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002423 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002424
2425 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2426 // (It's up the to the encoder to potentially drop the previous frame,
2427 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002428 timestamp_ms += kFrameIntervalMs;
2429 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2430 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002431 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002432
Asa Persson606d3cb2021-10-04 10:07:11 +02002433 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002434 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2435 // Frame size should be a multiple of the requested alignment.
2436 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2437 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2438 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2439 // Aspect ratio should match.
2440 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2441 codec.height * codec.simulcastStream[i].width);
2442 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002443
2444 video_stream_encoder_->Stop();
2445}
2446
Jonathan Yubc771b72017-12-08 17:04:29 -08002447TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2448 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002449 const int kWidth = 1280;
2450 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002451
2452 // We rely on the automatic resolution adaptation, but we handle framerate
2453 // adaptation manually by mocking the stats proxy.
2454 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002455
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002456 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002458 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002459 video_stream_encoder_->SetSource(&video_source_,
2460 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002461 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002462 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002463 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002464 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2465
Jonathan Yubc771b72017-12-08 17:04:29 -08002466 // Adapt down as far as possible.
2467 rtc::VideoSinkWants last_wants;
2468 int64_t t = 1;
2469 int loop_count = 0;
2470 do {
2471 ++loop_count;
2472 last_wants = video_source_.sink_wants();
2473
2474 // Simulate the framerate we've been asked to adapt to.
2475 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2476 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2477 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2478 mock_stats.input_frame_rate = fps;
2479 stats_proxy_->SetMockStats(mock_stats);
2480
2481 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2482 sink_.WaitForEncodedFrame(t);
2483 t += frame_interval_ms;
2484
mflodmancc3d4422017-08-03 08:27:51 -07002485 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002486 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002487 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002488 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2489 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002490 } while (video_source_.sink_wants().max_pixel_count <
2491 last_wants.max_pixel_count ||
2492 video_source_.sink_wants().max_framerate_fps <
2493 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002494
Jonathan Yubc771b72017-12-08 17:04:29 -08002495 // Verify that we've adapted all the way down.
2496 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002497 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002498 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2499 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002500 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002501 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2502 *video_source_.last_sent_height());
2503 EXPECT_EQ(kMinBalancedFramerateFps,
2504 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002505
Jonathan Yubc771b72017-12-08 17:04:29 -08002506 // Adapt back up the same number of times we adapted down.
2507 for (int i = 0; i < loop_count - 1; ++i) {
2508 last_wants = video_source_.sink_wants();
2509
2510 // Simulate the framerate we've been asked to adapt to.
2511 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2512 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2513 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2514 mock_stats.input_frame_rate = fps;
2515 stats_proxy_->SetMockStats(mock_stats);
2516
2517 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2518 sink_.WaitForEncodedFrame(t);
2519 t += frame_interval_ms;
2520
Henrik Boström91aa7322020-04-28 12:24:33 +02002521 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002522 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002523 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002524 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2525 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002526 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2527 last_wants.max_pixel_count ||
2528 video_source_.sink_wants().max_framerate_fps >
2529 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002530 }
2531
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002532 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002533 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002534 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002535 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2536 EXPECT_EQ((loop_count - 1) * 2,
2537 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002538
mflodmancc3d4422017-08-03 08:27:51 -07002539 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002540}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002541
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002542TEST_F(VideoStreamEncoderTest,
2543 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002544 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2545 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002546 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002547
2548 const int kFrameWidth = 1280;
2549 const int kFrameHeight = 720;
2550
2551 int64_t ntp_time = kFrameIntervalMs;
2552
2553 // Force an input frame rate to be available, or the adaptation call won't
2554 // know what framerate to adapt form.
2555 const int kInputFps = 30;
2556 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2557 stats.input_frame_rate = kInputFps;
2558 stats_proxy_->SetMockStats(stats);
2559
2560 video_source_.set_adaptation_enabled(true);
2561 video_stream_encoder_->SetSource(
2562 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002563 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002564 video_source_.IncomingCapturedFrame(
2565 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2566 sink_.WaitForEncodedFrame(ntp_time);
2567 ntp_time += kFrameIntervalMs;
2568
2569 // Trigger CPU overuse.
2570 video_stream_encoder_->TriggerCpuOveruse();
2571 video_source_.IncomingCapturedFrame(
2572 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2573 sink_.WaitForEncodedFrame(ntp_time);
2574 ntp_time += kFrameIntervalMs;
2575
2576 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2577 EXPECT_EQ(std::numeric_limits<int>::max(),
2578 video_source_.sink_wants().max_pixel_count);
2579 // Some framerate constraint should be set.
2580 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2581 EXPECT_LT(restricted_fps, kInputFps);
2582 video_source_.IncomingCapturedFrame(
2583 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2584 sink_.WaitForEncodedFrame(ntp_time);
2585 ntp_time += 100;
2586
Henrik Boström2671dac2020-05-19 16:29:09 +02002587 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002588 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2589 // Give the encoder queue time to process the change in degradation preference
2590 // by waiting for an encoded frame.
2591 video_source_.IncomingCapturedFrame(
2592 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2593 sink_.WaitForEncodedFrame(ntp_time);
2594 ntp_time += kFrameIntervalMs;
2595
2596 video_stream_encoder_->TriggerQualityLow();
2597 video_source_.IncomingCapturedFrame(
2598 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2599 sink_.WaitForEncodedFrame(ntp_time);
2600 ntp_time += kFrameIntervalMs;
2601
2602 // Some resolution constraint should be set.
2603 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2604 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2605 kFrameWidth * kFrameHeight);
2606 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2607
2608 int pixel_count = video_source_.sink_wants().max_pixel_count;
2609 // Triggering a CPU underuse should not change the sink wants since it has
2610 // not been overused for resolution since we changed degradation preference.
2611 video_stream_encoder_->TriggerCpuUnderuse();
2612 video_source_.IncomingCapturedFrame(
2613 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2614 sink_.WaitForEncodedFrame(ntp_time);
2615 ntp_time += kFrameIntervalMs;
2616 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2617 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2618
Evan Shrubsole64469032020-06-11 10:45:29 +02002619 // Change the degradation preference back. CPU underuse should not adapt since
2620 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002621 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002622 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2623 video_source_.IncomingCapturedFrame(
2624 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2625 sink_.WaitForEncodedFrame(ntp_time);
2626 ntp_time += 100;
2627 // Resolution adaptations is gone after changing degradation preference.
2628 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2629 EXPECT_EQ(std::numeric_limits<int>::max(),
2630 video_source_.sink_wants().max_pixel_count);
2631 // The fps adaptation from above is now back.
2632 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2633
2634 // Trigger CPU underuse.
2635 video_stream_encoder_->TriggerCpuUnderuse();
2636 video_source_.IncomingCapturedFrame(
2637 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2638 sink_.WaitForEncodedFrame(ntp_time);
2639 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002640 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2641
2642 // Trigger QP underuse, fps should return to normal.
2643 video_stream_encoder_->TriggerQualityHigh();
2644 video_source_.IncomingCapturedFrame(
2645 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2646 sink_.WaitForEncodedFrame(ntp_time);
2647 ntp_time += kFrameIntervalMs;
2648 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002649
2650 video_stream_encoder_->Stop();
2651}
2652
mflodmancc3d4422017-08-03 08:27:51 -07002653TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002654 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002655 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002656 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002657
sprangc5d62e22017-04-02 23:53:04 -07002658 const int kFrameWidth = 1280;
2659 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002660
Åsa Persson8c1bf952018-09-13 10:42:19 +02002661 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002662
kthelgason5e13d412016-12-01 03:59:51 -08002663 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002664 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002665 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002666 frame_timestamp += kFrameIntervalMs;
2667
perkj803d97f2016-11-01 11:45:46 -07002668 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002669 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002670 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002671 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002672 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002673 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002674
asapersson0944a802017-04-07 00:57:58 -07002675 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002676 // wanted resolution.
2677 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2678 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2679 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002680 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002681
2682 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002683 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002684 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002685 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002686 // Give the encoder queue time to process the change in degradation preference
2687 // by waiting for an encoded frame.
2688 new_video_source.IncomingCapturedFrame(
2689 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2690 sink_.WaitForEncodedFrame(frame_timestamp);
2691 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002692 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002693 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002694
sprangc5d62e22017-04-02 23:53:04 -07002695 // Force an input frame rate to be available, or the adaptation call won't
2696 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002697 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002698 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002699 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002700 stats_proxy_->SetMockStats(stats);
2701
mflodmancc3d4422017-08-03 08:27:51 -07002702 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002703 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002704 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002705 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002706 frame_timestamp += kFrameIntervalMs;
2707
2708 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002709 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002710 EXPECT_EQ(std::numeric_limits<int>::max(),
2711 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002712 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002713
asapersson02465b82017-04-10 01:12:52 -07002714 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002715 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2716 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002717 // Give the encoder queue time to process the change in degradation preference
2718 // by waiting for an encoded frame.
2719 new_video_source.IncomingCapturedFrame(
2720 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2721 sink_.WaitForEncodedFrame(frame_timestamp);
2722 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002723 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002724
mflodmancc3d4422017-08-03 08:27:51 -07002725 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002726 new_video_source.IncomingCapturedFrame(
2727 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002728 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002729 frame_timestamp += kFrameIntervalMs;
2730
2731 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002732 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002733
2734 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002735 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002736 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002737 // Give the encoder queue time to process the change in degradation preference
2738 // by waiting for an encoded frame.
2739 new_video_source.IncomingCapturedFrame(
2740 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2741 sink_.WaitForEncodedFrame(frame_timestamp);
2742 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002743 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2744 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002745 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002746 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002747
2748 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002750 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002751 // Give the encoder queue time to process the change in degradation preference
2752 // by waiting for an encoded frame.
2753 new_video_source.IncomingCapturedFrame(
2754 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2755 sink_.WaitForEncodedFrame(frame_timestamp);
2756 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002757 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2758 EXPECT_EQ(std::numeric_limits<int>::max(),
2759 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002760 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002761
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002763}
2764
mflodmancc3d4422017-08-03 08:27:51 -07002765TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002768
asaperssonfab67072017-04-04 05:51:49 -07002769 const int kWidth = 1280;
2770 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002771 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002772 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002773 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2774 EXPECT_FALSE(stats.bw_limited_resolution);
2775 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2776
2777 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002778 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002779 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002780 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002781
2782 stats = stats_proxy_->GetStats();
2783 EXPECT_TRUE(stats.bw_limited_resolution);
2784 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2785
2786 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002787 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002788 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002789 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002790
2791 stats = stats_proxy_->GetStats();
2792 EXPECT_FALSE(stats.bw_limited_resolution);
2793 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2794 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2795
mflodmancc3d4422017-08-03 08:27:51 -07002796 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002797}
2798
mflodmancc3d4422017-08-03 08:27:51 -07002799TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002802
2803 const int kWidth = 1280;
2804 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002805 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002806 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002807 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2808 EXPECT_FALSE(stats.cpu_limited_resolution);
2809 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2810
2811 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002812 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002813 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002814 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002815
2816 stats = stats_proxy_->GetStats();
2817 EXPECT_TRUE(stats.cpu_limited_resolution);
2818 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2819
2820 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002821 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002822 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002823 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002824
2825 stats = stats_proxy_->GetStats();
2826 EXPECT_FALSE(stats.cpu_limited_resolution);
2827 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002828 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002829
mflodmancc3d4422017-08-03 08:27:51 -07002830 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002831}
2832
mflodmancc3d4422017-08-03 08:27:51 -07002833TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002835 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002836
asaperssonfab67072017-04-04 05:51:49 -07002837 const int kWidth = 1280;
2838 const int kHeight = 720;
2839 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002840 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002841 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002842 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002843 EXPECT_FALSE(stats.cpu_limited_resolution);
2844 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2845
asaperssonfab67072017-04-04 05:51:49 -07002846 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002848 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002849 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002850 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002851 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002852 EXPECT_TRUE(stats.cpu_limited_resolution);
2853 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2854
2855 // Set new source with adaptation still enabled.
2856 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002858 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002859
asaperssonfab67072017-04-04 05:51:49 -07002860 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002861 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002862 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002863 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002864 EXPECT_TRUE(stats.cpu_limited_resolution);
2865 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2866
2867 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002868 video_stream_encoder_->SetSource(&new_video_source,
2869 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002870
asaperssonfab67072017-04-04 05:51:49 -07002871 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002872 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002873 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002874 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002875 EXPECT_FALSE(stats.cpu_limited_resolution);
2876 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2877
2878 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002879 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002880 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002881
asaperssonfab67072017-04-04 05:51:49 -07002882 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002883 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002884 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002885 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002886 EXPECT_TRUE(stats.cpu_limited_resolution);
2887 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2888
asaperssonfab67072017-04-04 05:51:49 -07002889 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002890 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002891 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002892 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002893 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002894 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002895 EXPECT_FALSE(stats.cpu_limited_resolution);
2896 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002897 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002898
mflodmancc3d4422017-08-03 08:27:51 -07002899 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002900}
2901
mflodmancc3d4422017-08-03 08:27:51 -07002902TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002903 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002904 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002905
asaperssonfab67072017-04-04 05:51:49 -07002906 const int kWidth = 1280;
2907 const int kHeight = 720;
2908 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002909 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002910 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002911 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002912 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002913 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002914
2915 // Set new source with adaptation still enabled.
2916 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002917 video_stream_encoder_->SetSource(&new_video_source,
2918 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002919
asaperssonfab67072017-04-04 05:51:49 -07002920 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002921 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002922 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002923 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002924 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002925 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002926
asaperssonfab67072017-04-04 05:51:49 -07002927 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002928 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002929 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002930 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002931 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002932 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002933 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002934 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002935
asaperssonfab67072017-04-04 05:51:49 -07002936 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002937 video_stream_encoder_->SetSource(&new_video_source,
2938 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002939
asaperssonfab67072017-04-04 05:51:49 -07002940 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002941 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002942 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002943 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002944 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002945 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002946
asapersson02465b82017-04-10 01:12:52 -07002947 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002948 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002949 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002950
asaperssonfab67072017-04-04 05:51:49 -07002951 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002952 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002953 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002954 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002955 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002956 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2957 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002958
mflodmancc3d4422017-08-03 08:27:51 -07002959 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002960}
2961
mflodmancc3d4422017-08-03 08:27:51 -07002962TEST_F(VideoStreamEncoderTest,
2963 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002965 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002966
2967 const int kWidth = 1280;
2968 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002969 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002970 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002971 video_source_.IncomingCapturedFrame(
2972 CreateFrame(timestamp_ms, kWidth, kHeight));
2973 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002974 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2975 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2976 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2977
2978 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002979 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002980 timestamp_ms += kFrameIntervalMs;
2981 video_source_.IncomingCapturedFrame(
2982 CreateFrame(timestamp_ms, kWidth, kHeight));
2983 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002984 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2985 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2986 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2987
2988 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002989 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002990 timestamp_ms += kFrameIntervalMs;
2991 video_source_.IncomingCapturedFrame(
2992 CreateFrame(timestamp_ms, kWidth, kHeight));
2993 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002994 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2995 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2996 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2997
Niels Möller4db138e2018-04-19 09:04:13 +02002998 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002999 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003000
3001 VideoEncoderConfig video_encoder_config;
3002 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3003 // Make format different, to force recreation of encoder.
3004 video_encoder_config.video_format.parameters["foo"] = "foo";
3005 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003006 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003007 timestamp_ms += kFrameIntervalMs;
3008 video_source_.IncomingCapturedFrame(
3009 CreateFrame(timestamp_ms, kWidth, kHeight));
3010 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003011 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3012 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3013 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3014
mflodmancc3d4422017-08-03 08:27:51 -07003015 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003016}
3017
mflodmancc3d4422017-08-03 08:27:51 -07003018TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003019 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003021 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003022
3023 const int kWidth = 1280;
3024 const int kHeight = 720;
3025 int sequence = 1;
3026
3027 // Enable BALANCED preference, no initial limitation.
3028 test::FrameForwarder source;
3029 video_stream_encoder_->SetSource(&source,
3030 webrtc::DegradationPreference::BALANCED);
3031 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3032 WaitForEncodedFrame(sequence++);
3033 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3034 EXPECT_FALSE(stats.cpu_limited_resolution);
3035 EXPECT_FALSE(stats.cpu_limited_framerate);
3036 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3037
3038 // Trigger CPU overuse, should now adapt down.
3039 video_stream_encoder_->TriggerCpuOveruse();
3040 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3041 WaitForEncodedFrame(sequence++);
3042 stats = stats_proxy_->GetStats();
3043 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3044
3045 // Set new degradation preference should clear restrictions since we changed
3046 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003047 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003048 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3049 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3050 WaitForEncodedFrame(sequence++);
3051 stats = stats_proxy_->GetStats();
3052 EXPECT_FALSE(stats.cpu_limited_resolution);
3053 EXPECT_FALSE(stats.cpu_limited_framerate);
3054 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3055
3056 // Force an input frame rate to be available, or the adaptation call won't
3057 // know what framerate to adapt from.
3058 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3059 mock_stats.input_frame_rate = 30;
3060 stats_proxy_->SetMockStats(mock_stats);
3061 video_stream_encoder_->TriggerCpuOveruse();
3062 stats_proxy_->ResetMockStats();
3063 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3064 WaitForEncodedFrame(sequence++);
3065
3066 // We have now adapted once.
3067 stats = stats_proxy_->GetStats();
3068 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3069
3070 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003071 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3072 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003073 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3074 WaitForEncodedFrame(sequence++);
3075 stats = stats_proxy_->GetStats();
3076 EXPECT_FALSE(stats.cpu_limited_resolution);
3077 EXPECT_FALSE(stats.cpu_limited_framerate);
3078 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3079
3080 video_stream_encoder_->Stop();
3081}
3082
3083TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003084 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003087
asapersson0944a802017-04-07 00:57:58 -07003088 const int kWidth = 1280;
3089 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003090 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003091
asaperssonfab67072017-04-04 05:51:49 -07003092 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003093 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003094 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003095 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003096 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003097 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3098
asapersson02465b82017-04-10 01:12:52 -07003099 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003100 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003101 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003102 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003103 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003104 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003105 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003106 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3107
3108 // Set new source with adaptation still enabled.
3109 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003110 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003111 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003112
3113 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003114 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003115 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003116 stats = stats_proxy_->GetStats();
3117 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003118 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003119 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3120
sprangc5d62e22017-04-02 23:53:04 -07003121 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003122 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003123 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003124 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003125 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003126 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003127 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003128 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003129 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003130 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003131 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3132
sprangc5d62e22017-04-02 23:53:04 -07003133 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003134 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003135 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3136 mock_stats.input_frame_rate = 30;
3137 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003138 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003139 stats_proxy_->ResetMockStats();
3140
3141 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003142 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003143 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003144
3145 // Framerate now adapted.
3146 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003147 EXPECT_FALSE(stats.cpu_limited_resolution);
3148 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003149 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3150
3151 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003152 video_stream_encoder_->SetSource(&new_video_source,
3153 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003154 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003155 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003156 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003157
3158 stats = stats_proxy_->GetStats();
3159 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003160 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003161 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3162
3163 // Try to trigger overuse. Should not succeed.
3164 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003165 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003166 stats_proxy_->ResetMockStats();
3167
3168 stats = stats_proxy_->GetStats();
3169 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003170 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003171 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3172
3173 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003174 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003175 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003176 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003177 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003178 stats = stats_proxy_->GetStats();
3179 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003180 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003181 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003182
3183 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003184 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003185 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003186 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003187 stats = stats_proxy_->GetStats();
3188 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003189 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003190 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3191
3192 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003194 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003195 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003196 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003197 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003198 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003199 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003200 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003201 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003202 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3203
3204 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003205 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003206 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003207 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003208 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003209 stats = stats_proxy_->GetStats();
3210 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003211 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003212 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003213 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003214
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003216}
3217
mflodmancc3d4422017-08-03 08:27:51 -07003218TEST_F(VideoStreamEncoderTest,
3219 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003220 const int kWidth = 1280;
3221 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003222 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003223 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003224
asaperssonfab67072017-04-04 05:51:49 -07003225 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003226 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003227
asaperssonfab67072017-04-04 05:51:49 -07003228 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003229 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003230
asaperssonfab67072017-04-04 05:51:49 -07003231 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003232 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003233
asaperssonfab67072017-04-04 05:51:49 -07003234 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003235 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003236
kthelgason876222f2016-11-29 01:44:11 -08003237 // Expect a scale down.
3238 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003239 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003240
asapersson02465b82017-04-10 01:12:52 -07003241 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003242 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003243 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003244 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003245
asaperssonfab67072017-04-04 05:51:49 -07003246 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003247 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003248 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003249 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003250
asaperssonfab67072017-04-04 05:51:49 -07003251 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003252 EXPECT_EQ(std::numeric_limits<int>::max(),
3253 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003254
asaperssonfab67072017-04-04 05:51:49 -07003255 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003256 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003257 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003258 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003259
asapersson02465b82017-04-10 01:12:52 -07003260 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003261 EXPECT_EQ(std::numeric_limits<int>::max(),
3262 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003263
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003265}
3266
mflodmancc3d4422017-08-03 08:27:51 -07003267TEST_F(VideoStreamEncoderTest,
3268 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003269 const int kWidth = 1280;
3270 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003271 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003272 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003273
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003274 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003275 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003277 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003278
3279 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003280 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003281 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003282 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3283 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3284
3285 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003286 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003287 EXPECT_THAT(source.sink_wants(),
3288 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003289 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3290 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3291 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3292
3293 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003294 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003295 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3296 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3297 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3298
mflodmancc3d4422017-08-03 08:27:51 -07003299 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003300}
3301
mflodmancc3d4422017-08-03 08:27:51 -07003302TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003303 const int kWidth = 1280;
3304 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003306 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003307
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003308 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003309 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003310 video_stream_encoder_->SetSource(&source,
3311 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003312 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3313 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003314 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003315
3316 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003317 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003318 EXPECT_THAT(source.sink_wants(),
3319 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003320 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3321 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3322 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3323
3324 // Trigger adapt down for same input resolution, expect no change.
3325 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3326 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003327 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003328 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3329 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3330 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3331
3332 // Trigger adapt down for larger input resolution, expect no change.
3333 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3334 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003335 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003336 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3337 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3338 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3339
mflodmancc3d4422017-08-03 08:27:51 -07003340 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003341}
3342
mflodmancc3d4422017-08-03 08:27:51 -07003343TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003344 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3345 const int kWidth = 640;
3346 const int kHeight = 360;
3347 const int64_t kFrameIntervalMs = 150;
3348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003350
3351 // Enable BALANCED preference, no initial limitation.
3352 AdaptingFrameForwarder source(&time_controller_);
3353 source.set_adaptation_enabled(true);
3354 video_stream_encoder_->SetSource(&source,
3355 webrtc::DegradationPreference::BALANCED);
3356
3357 int64_t timestamp_ms = kFrameIntervalMs;
3358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3359 sink_.WaitForEncodedFrame(kWidth, kHeight);
3360 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3361 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3362 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3363 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3364
3365 // Trigger adapt down, expect reduced fps (640x360@15fps).
3366 video_stream_encoder_->TriggerQualityLow();
3367 timestamp_ms += kFrameIntervalMs;
3368 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3369 sink_.WaitForEncodedFrame(timestamp_ms);
3370 EXPECT_THAT(source.sink_wants(),
3371 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3372 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3373 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3374 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3375
3376 // Source requests 270p, expect reduced resolution (480x270@15fps).
3377 source.OnOutputFormatRequest(480, 270);
3378 timestamp_ms += kFrameIntervalMs;
3379 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3380 WaitForEncodedFrame(480, 270);
3381 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Trigger adapt down, expect reduced fps (480x270@10fps).
3384 video_stream_encoder_->TriggerQualityLow();
3385 timestamp_ms += kFrameIntervalMs;
3386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3387 sink_.WaitForEncodedFrame(timestamp_ms);
3388 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3389 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3390 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3391 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3392
3393 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3394 source.OnOutputFormatRequest(320, 180);
3395 timestamp_ms += kFrameIntervalMs;
3396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3397 WaitForEncodedFrame(320, 180);
3398 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Trigger adapt down, expect reduced fps (320x180@7fps).
3401 video_stream_encoder_->TriggerQualityLow();
3402 timestamp_ms += kFrameIntervalMs;
3403 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3404 sink_.WaitForEncodedFrame(timestamp_ms);
3405 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3406 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3407 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3408 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3409
3410 // Source requests VGA, expect increased resolution (640x360@7fps).
3411 source.OnOutputFormatRequest(640, 360);
3412 timestamp_ms += kFrameIntervalMs;
3413 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3414 WaitForEncodedFrame(timestamp_ms);
3415 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
3417 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3418 video_stream_encoder_->TriggerQualityHigh();
3419 timestamp_ms += kFrameIntervalMs;
3420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3421 WaitForEncodedFrame(timestamp_ms);
3422 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3425 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
3427 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3428 video_stream_encoder_->TriggerQualityHigh();
3429 timestamp_ms += kFrameIntervalMs;
3430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3431 WaitForEncodedFrame(timestamp_ms);
3432 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3435 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 // Trigger adapt up, expect increased fps (640x360@maxfps).
3438 video_stream_encoder_->TriggerQualityHigh();
3439 timestamp_ms += kFrameIntervalMs;
3440 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3441 WaitForEncodedFrame(timestamp_ms);
3442 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3444 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3445 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3446
3447 video_stream_encoder_->Stop();
3448}
3449
3450TEST_F(VideoStreamEncoderTest,
3451 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3452 const int kWidth = 1280;
3453 const int kHeight = 720;
3454 const int64_t kFrameIntervalMs = 150;
3455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003456 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003457
3458 // Enable BALANCED preference, no initial limitation.
3459 AdaptingFrameForwarder source(&time_controller_);
3460 source.set_adaptation_enabled(true);
3461 video_stream_encoder_->SetSource(&source,
3462 webrtc::DegradationPreference::BALANCED);
3463
3464 int64_t timestamp_ms = kFrameIntervalMs;
3465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3466 sink_.WaitForEncodedFrame(kWidth, kHeight);
3467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3470 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3471
3472 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3473 video_stream_encoder_->TriggerQualityLow();
3474 timestamp_ms += kFrameIntervalMs;
3475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3476 sink_.WaitForEncodedFrame(timestamp_ms);
3477 EXPECT_THAT(source.sink_wants(),
3478 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3479 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3481 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3482
3483 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3484 video_stream_encoder_->TriggerQualityLow();
3485 timestamp_ms += kFrameIntervalMs;
3486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3487 sink_.WaitForEncodedFrame(timestamp_ms);
3488 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3490 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3491 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Trigger adapt down, expect reduced fps (640x360@15fps).
3494 video_stream_encoder_->TriggerQualityLow();
3495 timestamp_ms += kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3497 WaitForEncodedFrame(timestamp_ms);
3498 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3501 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3502
3503 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3504 source.OnOutputFormatRequest(320, 180);
3505 timestamp_ms += kFrameIntervalMs;
3506 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3507 WaitForEncodedFrame(320, 180);
3508 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3509 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3510
3511 // Trigger adapt down, expect reduced fps (320x180@7fps).
3512 video_stream_encoder_->TriggerCpuOveruse();
3513 timestamp_ms += kFrameIntervalMs;
3514 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3515 WaitForEncodedFrame(timestamp_ms);
3516 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3519 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3520 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3521 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3522 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3523
3524 // Source requests HD, expect increased resolution (640x360@7fps).
3525 source.OnOutputFormatRequest(1280, 720);
3526 timestamp_ms += kFrameIntervalMs;
3527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3528 WaitForEncodedFrame(timestamp_ms);
3529 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3530 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3531
3532 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3533 video_stream_encoder_->TriggerCpuUnderuse();
3534 timestamp_ms += kFrameIntervalMs;
3535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3536 WaitForEncodedFrame(timestamp_ms);
3537 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3541 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3542 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3543 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3544
3545 // Trigger adapt up, expect increased fps (640x360@maxfps).
3546 video_stream_encoder_->TriggerQualityHigh();
3547 video_stream_encoder_->TriggerCpuUnderuse();
3548 timestamp_ms += kFrameIntervalMs;
3549 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3550 WaitForEncodedFrame(timestamp_ms);
3551 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3555 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3556 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3557 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3558
3559 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3560 video_stream_encoder_->TriggerQualityHigh();
3561 video_stream_encoder_->TriggerCpuUnderuse();
3562 timestamp_ms += kFrameIntervalMs;
3563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3564 WaitForEncodedFrame(timestamp_ms);
3565 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3568 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3569 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3570 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3571 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3572
3573 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3574 video_stream_encoder_->TriggerQualityHigh();
3575 video_stream_encoder_->TriggerCpuUnderuse();
3576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 WaitForEncodedFrame(timestamp_ms);
3579 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3580 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3582 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3583 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3584 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3586
3587 video_stream_encoder_->Stop();
3588}
3589
3590TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003591 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003592 const int kWidth = 1280;
3593 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003595 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003596
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003597 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003598 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003599 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003600 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003601
3602 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003603 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003604 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003605 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3606 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3607
3608 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003609 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003610 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003611 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3612 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3613
mflodmancc3d4422017-08-03 08:27:51 -07003614 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003615}
3616
mflodmancc3d4422017-08-03 08:27:51 -07003617TEST_F(VideoStreamEncoderTest,
3618 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003619 const int kWidth = 1280;
3620 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003622 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003623
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003624 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003625 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003626 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003627 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003628
3629 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003630 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003631 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003632 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003633 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3634
3635 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003636 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003637 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003639 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3640
mflodmancc3d4422017-08-03 08:27:51 -07003641 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003642}
3643
mflodmancc3d4422017-08-03 08:27:51 -07003644TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003645 const int kWidth = 1280;
3646 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003648 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003649
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003650 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003651 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003652 video_stream_encoder_->SetSource(&source,
3653 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003654
3655 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3656 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003657 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3660 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3661
3662 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003663 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003664 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003665 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3666 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3668
mflodmancc3d4422017-08-03 08:27:51 -07003669 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003670}
3671
mflodmancc3d4422017-08-03 08:27:51 -07003672TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003673 const int kWidth = 1280;
3674 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003676 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003677
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003678 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003679 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003680 video_stream_encoder_->SetSource(&source,
3681 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003682
3683 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3684 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003686 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3687 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3688 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3689
3690 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003691 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003692 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003693 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3694 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3695 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3696
mflodmancc3d4422017-08-03 08:27:51 -07003697 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003698}
3699
mflodmancc3d4422017-08-03 08:27:51 -07003700TEST_F(VideoStreamEncoderTest,
3701 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003702 const int kWidth = 1280;
3703 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003705 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003706
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003707 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003708 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003709 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003710 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003711 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003712
3713 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003714 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003715 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3718
3719 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003720 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003721 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003722 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003723 EXPECT_THAT(source.sink_wants(),
3724 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003725 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3726 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3727
3728 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003729 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003730 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3732 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3733 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3734
mflodmancc3d4422017-08-03 08:27:51 -07003735 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003736}
3737
mflodmancc3d4422017-08-03 08:27:51 -07003738TEST_F(VideoStreamEncoderTest,
3739 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003740 const int kWidth = 1280;
3741 const int kHeight = 720;
3742 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003745
3746 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3747 stats.input_frame_rate = kInputFps;
3748 stats_proxy_->SetMockStats(stats);
3749
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003750 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003751 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3752 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003753 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003754
3755 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003756 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003757 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3758 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(video_source_.sink_wants(),
3760 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003761
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003762 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003763 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003764 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003765 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003766 // Give the encoder queue time to process the change in degradation preference
3767 // by waiting for an encoded frame.
3768 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3769 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003770 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003771
3772 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003774 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3775 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003776 EXPECT_THAT(new_video_source.sink_wants(),
3777 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003778
3779 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003780 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003781 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003782
mflodmancc3d4422017-08-03 08:27:51 -07003783 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003784}
3785
mflodmancc3d4422017-08-03 08:27:51 -07003786TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003787 const int kWidth = 1280;
3788 const int kHeight = 720;
3789 const size_t kNumFrames = 10;
3790
Henrik Boström381d1092020-05-12 18:49:07 +02003791 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003792 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003793
asaperssond0de2952017-04-21 01:47:31 -07003794 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003795 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003796 video_source_.set_adaptation_enabled(true);
3797
3798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3799 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3800
3801 int downscales = 0;
3802 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003803 video_source_.IncomingCapturedFrame(
3804 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3805 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003806
asaperssonfab67072017-04-04 05:51:49 -07003807 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003808 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003809 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003810 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003811
3812 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3813 ++downscales;
3814
3815 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3816 EXPECT_EQ(downscales,
3817 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3818 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003819 }
mflodmancc3d4422017-08-03 08:27:51 -07003820 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003821}
3822
mflodmancc3d4422017-08-03 08:27:51 -07003823TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003824 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3825 const int kWidth = 1280;
3826 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003828 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003830 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003831 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003832 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003833 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003834 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003835
Åsa Persson8c1bf952018-09-13 10:42:19 +02003836 int64_t timestamp_ms = kFrameIntervalMs;
3837 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003838 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003839 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003840 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3841 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3842
3843 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003844 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003845 timestamp_ms += kFrameIntervalMs;
3846 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3847 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003848 EXPECT_THAT(source.sink_wants(),
3849 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003850 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3851 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3852
3853 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003854 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003855 timestamp_ms += kFrameIntervalMs;
3856 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003857 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003858 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003859 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3860 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3861
3862 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003863 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003864 timestamp_ms += kFrameIntervalMs;
3865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3866 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003867 EXPECT_THAT(source.sink_wants(),
3868 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003869 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3870 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3871
3872 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003873 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003874 timestamp_ms += kFrameIntervalMs;
3875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003876 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003877 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003878 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3879 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3880
mflodmancc3d4422017-08-03 08:27:51 -07003881 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003882}
3883
mflodmancc3d4422017-08-03 08:27:51 -07003884TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003885 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3886 const int kWidth = 1280;
3887 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003890
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003891 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003892 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003893 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003894 video_stream_encoder_->SetSource(&source,
3895 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003896
Åsa Persson8c1bf952018-09-13 10:42:19 +02003897 int64_t timestamp_ms = kFrameIntervalMs;
3898 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003899 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003900 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003901 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3902 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3903
3904 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003905 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3908 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003909 EXPECT_THAT(source.sink_wants(),
3910 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003911 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3912 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913
3914 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003915 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003916 timestamp_ms += kFrameIntervalMs;
3917 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003918 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3921 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3922
3923 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003924 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003925 timestamp_ms += kFrameIntervalMs;
3926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3927 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003928 EXPECT_THAT(source.sink_wants(),
3929 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003930 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3931 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3932
3933 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003934 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003935 timestamp_ms += kFrameIntervalMs;
3936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003937 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003939 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3940 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3941
mflodmancc3d4422017-08-03 08:27:51 -07003942 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003943}
3944
Sergey Silkin41c650b2019-10-14 13:12:19 +02003945TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3946 fake_encoder_.SetResolutionBitrateLimits(
3947 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3948
Henrik Boström381d1092020-05-12 18:49:07 +02003949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003950 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3951 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3952 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3953 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003954
3955 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003956 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003957 source.set_adaptation_enabled(true);
3958 video_stream_encoder_->SetSource(
3959 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3960
3961 // Insert 720p frame.
3962 int64_t timestamp_ms = kFrameIntervalMs;
3963 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3964 WaitForEncodedFrame(1280, 720);
3965
3966 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003968 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3969 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3970 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3971 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003972 video_stream_encoder_->TriggerQualityLow();
3973
3974 // Insert 720p frame. It should be downscaled and encoded.
3975 timestamp_ms += kFrameIntervalMs;
3976 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3977 WaitForEncodedFrame(960, 540);
3978
3979 // Trigger adapt up. Higher resolution should not be requested duo to lack
3980 // of bitrate.
3981 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003982 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003983
3984 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003986 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3987 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3988 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3989 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003990
3991 // Trigger adapt up. Higher resolution should be requested.
3992 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003993 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003994
3995 video_stream_encoder_->Stop();
3996}
3997
3998TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3999 fake_encoder_.SetResolutionBitrateLimits(
4000 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4001
4002 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004004 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4005 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4006 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4007 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004008
4009 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004010 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004011 source.set_adaptation_enabled(true);
4012 video_stream_encoder_->SetSource(
4013 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4014
4015 // Insert 720p frame. It should be dropped and lower resolution should be
4016 // requested.
4017 int64_t timestamp_ms = kFrameIntervalMs;
4018 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4019 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004020 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004021
4022 // Insert 720p frame. It should be downscaled and encoded.
4023 timestamp_ms += kFrameIntervalMs;
4024 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4025 WaitForEncodedFrame(960, 540);
4026
4027 video_stream_encoder_->Stop();
4028}
4029
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004030class BalancedDegradationTest : public VideoStreamEncoderTest {
4031 protected:
4032 void SetupTest() {
4033 // Reset encoder for field trials to take effect.
4034 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004035 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004036
4037 // Enable BALANCED preference.
4038 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004039 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4040 }
4041
Asa Persson606d3cb2021-10-04 10:07:11 +02004042 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004044 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004045 }
4046
Åsa Persson45b176f2019-09-30 11:19:05 +02004047 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004048 timestamp_ms_ += kFrameIntervalMs;
4049 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004050 }
4051
4052 void InsertFrameAndWaitForEncoded() {
4053 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004054 sink_.WaitForEncodedFrame(timestamp_ms_);
4055 }
4056
4057 const int kWidth = 640; // pixels:640x360=230400
4058 const int kHeight = 360;
4059 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4060 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004061 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004062};
4063
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004064TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004065 test::ScopedFieldTrials field_trials(
4066 "WebRTC-Video-BalancedDegradationSettings/"
4067 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4068 SetupTest();
4069
4070 // Force input frame rate.
4071 const int kInputFps = 24;
4072 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4073 stats.input_frame_rate = kInputFps;
4074 stats_proxy_->SetMockStats(stats);
4075
Åsa Persson45b176f2019-09-30 11:19:05 +02004076 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004077 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004078
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004079 // Trigger adapt down, expect scaled down framerate and resolution,
4080 // since Fps diff (input-requested:0) < threshold.
4081 video_stream_encoder_->TriggerQualityLow();
4082 EXPECT_THAT(source_.sink_wants(),
4083 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004084
4085 video_stream_encoder_->Stop();
4086}
4087
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004088TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004089 test::ScopedFieldTrials field_trials(
4090 "WebRTC-Video-BalancedDegradationSettings/"
4091 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4092 SetupTest();
4093
4094 // Force input frame rate.
4095 const int kInputFps = 25;
4096 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4097 stats.input_frame_rate = kInputFps;
4098 stats_proxy_->SetMockStats(stats);
4099
Åsa Persson45b176f2019-09-30 11:19:05 +02004100 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004101 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004102
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004103 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4104 // Fps diff (input-requested:1) == threshold.
4105 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004106 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004107
4108 video_stream_encoder_->Stop();
4109}
4110
4111TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4112 test::ScopedFieldTrials field_trials(
4113 "WebRTC-Video-BalancedDegradationSettings/"
4114 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4115 SetupTest();
4116
4117 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4118
Åsa Persson45b176f2019-09-30 11:19:05 +02004119 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004120 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004121
4122 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4123 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004124 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004125
4126 video_stream_encoder_->Stop();
4127}
4128
Åsa Perssonccfb3402019-09-25 15:13:04 +02004129TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004130 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004131 "WebRTC-Video-BalancedDegradationSettings/"
4132 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004133 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004134
Asa Persson606d3cb2021-10-04 10:07:11 +02004135 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4136 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4137 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004138
Åsa Persson45b176f2019-09-30 11:19:05 +02004139 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004140 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004141 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4142
4143 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4144 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004145 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004146 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004147 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4148
4149 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4150 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004151 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004152 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004153 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4154
Åsa Persson30ab0152019-08-27 12:22:33 +02004155 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4156 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004157 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004158 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004159 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004160 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4161
4162 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004163 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004164 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004165 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004166
Åsa Persson30ab0152019-08-27 12:22:33 +02004167 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004168 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004169 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004170 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004171 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004172 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4173
4174 video_stream_encoder_->Stop();
4175}
4176
Åsa Perssonccfb3402019-09-25 15:13:04 +02004177TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004178 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4179 test::ScopedFieldTrials field_trials(
4180 "WebRTC-Video-BalancedDegradationSettings/"
4181 "pixels:57600|129600|230400,fps:7|24|24/");
4182 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004183 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004184
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004185 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004186
4187 // Insert frame, expect scaled down:
4188 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4189 InsertFrame();
4190 EXPECT_FALSE(WaitForFrame(1000));
4191 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4192 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4193
4194 // Insert frame, expect scaled down:
4195 // resolution (320x180@24fps).
4196 InsertFrame();
4197 EXPECT_FALSE(WaitForFrame(1000));
4198 EXPECT_LT(source_.sink_wants().max_pixel_count,
4199 source_.last_wants().max_pixel_count);
4200 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4201
4202 // Frame should not be dropped (min pixels per frame reached).
4203 InsertFrameAndWaitForEncoded();
4204
4205 video_stream_encoder_->Stop();
4206}
4207
4208TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004209 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004210 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004211 "WebRTC-Video-BalancedDegradationSettings/"
4212 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004213 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004214
Asa Persson606d3cb2021-10-04 10:07:11 +02004215 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4216 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4217 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004218
Åsa Persson45b176f2019-09-30 11:19:05 +02004219 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004220 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004221 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4222
4223 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4224 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004225 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004226 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004227 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4228
4229 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4230 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004231 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004232 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004233 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4234
4235 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4236 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004237 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004238 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004239 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4240
Åsa Persson30ab0152019-08-27 12:22:33 +02004241 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4242 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004243 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004244 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004245 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4246
4247 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4248 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004249 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004250 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4251
4252 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004253 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004254 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004255 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004256 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004257 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4258
4259 video_stream_encoder_->Stop();
4260}
4261
Åsa Perssonccfb3402019-09-25 15:13:04 +02004262TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004263 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004264 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004265 "WebRTC-Video-BalancedDegradationSettings/"
4266 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004267 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004268
Asa Persson606d3cb2021-10-04 10:07:11 +02004269 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4270 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4271 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4272 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4273 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004274
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004276 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004277 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4278
4279 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4280 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004281 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004282 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004283 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4284
4285 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4286 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004287 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004288 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004289 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4290
4291 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4292 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004293 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004294 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004295 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4296
4297 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4298 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004299 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004300 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4301
4302 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004303 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004304 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004305 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004306 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4308
4309 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004310 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004311 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004312 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004313 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4314
4315 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004316 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004317 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004318 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004319 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004320 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4321
Åsa Persson1b247f12019-08-14 17:26:39 +02004322 video_stream_encoder_->Stop();
4323}
4324
mflodmancc3d4422017-08-03 08:27:51 -07004325TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004326 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4327 const int kWidth = 1280;
4328 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004329 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004330 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004331
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004332 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004333 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004334 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004335 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004336 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004337
Åsa Persson8c1bf952018-09-13 10:42:19 +02004338 int64_t timestamp_ms = kFrameIntervalMs;
4339 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004340 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004341 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004342 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4344 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4345 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4346
4347 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004348 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004349 timestamp_ms += kFrameIntervalMs;
4350 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4351 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004352 EXPECT_THAT(source.sink_wants(),
4353 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004354 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4355 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4356 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4357 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4358
4359 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004360 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004361 timestamp_ms += kFrameIntervalMs;
4362 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4363 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004364 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004365 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4366 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4367 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4368 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4369
Jonathan Yubc771b72017-12-08 17:04:29 -08004370 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004371 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004372 timestamp_ms += kFrameIntervalMs;
4373 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4374 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004375 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004376 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4377 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004378 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004379 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4380
Jonathan Yubc771b72017-12-08 17:04:29 -08004381 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004382 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004383 timestamp_ms += kFrameIntervalMs;
4384 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4385 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004386 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004387 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004388 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4389 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4390 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4391 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4392
Jonathan Yubc771b72017-12-08 17:04:29 -08004393 // Trigger quality adapt down, expect no change (min resolution reached).
4394 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004395 timestamp_ms += kFrameIntervalMs;
4396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4397 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004398 EXPECT_THAT(source.sink_wants(), FpsMax());
4399 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004400 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4401 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4402 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4403 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4404
Evan Shrubsole64469032020-06-11 10:45:29 +02004405 // Trigger quality adapt up, expect upscaled resolution (480x270).
4406 video_stream_encoder_->TriggerQualityHigh();
4407 timestamp_ms += kFrameIntervalMs;
4408 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4409 WaitForEncodedFrame(timestamp_ms);
4410 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4411 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4412 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4414 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
4416 // Trigger quality and cpu adapt up since both are most limited, expect
4417 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004418 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004419 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004420 timestamp_ms += kFrameIntervalMs;
4421 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4422 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004423 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004424 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4426 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004427 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004428
Evan Shrubsole64469032020-06-11 10:45:29 +02004429 // Trigger quality and cpu adapt up since both are most limited, expect
4430 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004431 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004432 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004433 timestamp_ms += kFrameIntervalMs;
4434 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4435 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004436 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004437 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004438 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004439 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004440 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4441 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004442
Evan Shrubsole64469032020-06-11 10:45:29 +02004443 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4444 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004445 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004446 timestamp_ms += kFrameIntervalMs;
4447 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4448 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004449 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004450 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4451 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004452 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004453 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004454
4455 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004456 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004457 timestamp_ms += kFrameIntervalMs;
4458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004459 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004460 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004461 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004462 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004464 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004465 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004466
mflodmancc3d4422017-08-03 08:27:51 -07004467 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004468}
4469
mflodmancc3d4422017-08-03 08:27:51 -07004470TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004471 const int kWidth = 640;
4472 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004473
Henrik Boström381d1092020-05-12 18:49:07 +02004474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004475 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004476
perkj803d97f2016-11-01 11:45:46 -07004477 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004478 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004479 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004480 }
4481
mflodmancc3d4422017-08-03 08:27:51 -07004482 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004483 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004484 video_source_.IncomingCapturedFrame(CreateFrame(
4485 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004486 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004487 }
4488
mflodmancc3d4422017-08-03 08:27:51 -07004489 video_stream_encoder_->Stop();
4490 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004491 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004492
Ying Wangef3998f2019-12-09 13:06:53 +01004493 EXPECT_METRIC_EQ(
4494 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4495 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004496 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4497}
4498
mflodmancc3d4422017-08-03 08:27:51 -07004499TEST_F(VideoStreamEncoderTest,
4500 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004501 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004502 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004503 const int kWidth = 640;
4504 const int kHeight = 360;
4505
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004506 video_stream_encoder_->SetSource(&video_source_,
4507 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004508
4509 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4510 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004511 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004512 }
4513
mflodmancc3d4422017-08-03 08:27:51 -07004514 video_stream_encoder_->Stop();
4515 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004516 stats_proxy_.reset();
4517
4518 EXPECT_EQ(0,
4519 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4520}
4521
Per Kjellanderdcef6412020-10-07 15:09:05 +02004522TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4523 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004524 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004525 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004526
4527 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004528 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004529 SimulcastRateAllocator(fake_encoder_.config())
4530 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004531 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004532
Henrik Boström381d1092020-05-12 18:49:07 +02004533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004534 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004535
sprang57c2fff2017-01-16 06:24:02 -08004536 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004537 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4538 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004539 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4540 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4541
Erik Språngd7329ca2019-02-21 21:19:53 +01004542 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004543 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004544 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004545
Per Kjellanderdcef6412020-10-07 15:09:05 +02004546 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004547 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004548 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4549 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004550 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004551 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004552
Per Kjellanderdcef6412020-10-07 15:09:05 +02004553 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004554 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004555 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004556 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004557 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4558 WaitForEncodedFrame(CurrentTimeMs());
4559 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004560 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004561 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004562
mflodmancc3d4422017-08-03 08:27:51 -07004563 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004564}
4565
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004566TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004567 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004568 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004569 kVideoLayersAllocation);
4570
4571 const int kDefaultFps = 30;
4572
4573 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004574 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004575
4576 video_source_.IncomingCapturedFrame(
4577 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4578 WaitForEncodedFrame(CurrentTimeMs());
4579 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4580 VideoLayersAllocation last_layer_allocation =
4581 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004582 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004583 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4584
4585 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004586 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004587 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004588 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004589 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4590
Erik Språng9d69cbe2020-10-22 17:44:42 +02004591 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004592 int number_of_layers_allocation = 1;
4593 const int64_t start_time_ms = CurrentTimeMs();
4594 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4595 video_source_.IncomingCapturedFrame(
4596 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4597 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004598 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4599 number_of_layers_allocation = sink_.number_of_layers_allocations();
4600 VideoLayersAllocation new_allocation =
4601 sink_.GetLastVideoLayersAllocation();
4602 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4603 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4604 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4605 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4606 .target_bitrate_per_temporal_layer,
4607 last_layer_allocation.active_spatial_layers[0]
4608 .target_bitrate_per_temporal_layer);
4609 last_layer_allocation = new_allocation;
4610 }
4611 }
4612 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4613 video_stream_encoder_->Stop();
4614}
4615
4616TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004617 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004618 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4619 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4620 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004621 VideoEncoderConfig video_encoder_config;
4622 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4623 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004624 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004625 video_encoder_config.content_type =
4626 VideoEncoderConfig::ContentType::kRealtimeVideo;
4627 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004628 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004629 VideoEncoder::GetDefaultVp8Settings());
4630 for (auto& layer : video_encoder_config.simulcast_layers) {
4631 layer.num_temporal_layers = 2;
4632 }
4633 // Simulcast layers are used for enabling/disabling streams.
4634 video_encoder_config.simulcast_layers[0].active = true;
4635 video_encoder_config.simulcast_layers[1].active = false;
4636 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004637 ConfigureEncoder(std::move(video_encoder_config),
4638 VideoStreamEncoder::BitrateAllocationCallbackType::
4639 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004640
4641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004642 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004643
4644 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4645 WaitForEncodedFrame(CurrentTimeMs());
4646 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4647 VideoLayersAllocation last_layer_allocation =
4648 sink_.GetLastVideoLayersAllocation();
4649
4650 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4651 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4652 .target_bitrate_per_temporal_layer,
4653 SizeIs(2));
4654 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4655 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4656 video_stream_encoder_->Stop();
4657}
4658
4659TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004660 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004661 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4662 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4663 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004664 VideoEncoderConfig video_encoder_config;
4665 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4666 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004667 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004668 video_encoder_config.content_type =
4669 VideoEncoderConfig::ContentType::kRealtimeVideo;
4670 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004671 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004672 VideoEncoder::GetDefaultVp8Settings());
4673 for (auto& layer : video_encoder_config.simulcast_layers) {
4674 layer.num_temporal_layers = 2;
4675 }
4676 // Simulcast layers are used for enabling/disabling streams.
4677 video_encoder_config.simulcast_layers[0].active = true;
4678 video_encoder_config.simulcast_layers[1].active = false;
4679 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004680 ConfigureEncoder(std::move(video_encoder_config),
4681 VideoStreamEncoder::BitrateAllocationCallbackType::
4682 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004683
4684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004685 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004686
4687 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4688 WaitForEncodedFrame(CurrentTimeMs());
4689 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4690 VideoLayersAllocation last_layer_allocation =
4691 sink_.GetLastVideoLayersAllocation();
4692
4693 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4694 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4695 .target_bitrate_per_temporal_layer,
4696 SizeIs(2));
4697 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4698
4699 video_stream_encoder_->Stop();
4700}
4701
4702TEST_F(VideoStreamEncoderTest,
4703 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4704 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4705 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004706 VideoEncoderConfig video_encoder_config;
4707 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4708 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004709 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004710 video_encoder_config.content_type =
4711 VideoEncoderConfig::ContentType::kRealtimeVideo;
4712 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4713 vp9_settings.numberOfSpatialLayers = 2;
4714 vp9_settings.numberOfTemporalLayers = 2;
4715 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4716 vp9_settings.automaticResizeOn = false;
4717 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004718 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004719 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004720 ConfigureEncoder(std::move(video_encoder_config),
4721 VideoStreamEncoder::BitrateAllocationCallbackType::
4722 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004723
4724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004725 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004726
4727 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4728 WaitForEncodedFrame(CurrentTimeMs());
4729 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4730 VideoLayersAllocation last_layer_allocation =
4731 sink_.GetLastVideoLayersAllocation();
4732
4733 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4734 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4735 .target_bitrate_per_temporal_layer,
4736 SizeIs(2));
4737 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4738 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4739 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4740 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4741 .target_bitrate_per_temporal_layer,
4742 SizeIs(2));
4743 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4744 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4745 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4746
4747 // Since full SVC is used, expect the top layer to utilize the full target
4748 // rate.
4749 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4750 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004751 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752 video_stream_encoder_->Stop();
4753}
4754
4755TEST_F(VideoStreamEncoderTest,
4756 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4757 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4758 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004759 VideoEncoderConfig video_encoder_config;
4760 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4761 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004762 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004763 video_encoder_config.content_type =
4764 VideoEncoderConfig::ContentType::kRealtimeVideo;
4765 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4766 vp9_settings.numberOfSpatialLayers = 2;
4767 vp9_settings.numberOfTemporalLayers = 2;
4768 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4769 vp9_settings.automaticResizeOn = false;
4770 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004771 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004773 ConfigureEncoder(std::move(video_encoder_config),
4774 VideoStreamEncoder::BitrateAllocationCallbackType::
4775 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004776
4777 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004778 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004779
4780 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4781 WaitForEncodedFrame(CurrentTimeMs());
4782 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4783 VideoLayersAllocation last_layer_allocation =
4784 sink_.GetLastVideoLayersAllocation();
4785
4786 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4787 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4788 .target_bitrate_per_temporal_layer,
4789 SizeIs(1));
4790 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4791 .target_bitrate_per_temporal_layer,
4792 SizeIs(1));
4793 // Since full SVC is used, expect the top layer to utilize the full target
4794 // rate.
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4796 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004797 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004798 video_stream_encoder_->Stop();
4799}
4800
4801TEST_F(VideoStreamEncoderTest,
4802 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4803 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4804 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004805 VideoEncoderConfig video_encoder_config;
4806 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4807 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004808 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004809 video_encoder_config.content_type =
4810 VideoEncoderConfig::ContentType::kRealtimeVideo;
4811 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4812 vp9_settings.numberOfSpatialLayers = 2;
4813 vp9_settings.numberOfTemporalLayers = 2;
4814 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4815 vp9_settings.automaticResizeOn = false;
4816 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004817 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004818 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004819 ConfigureEncoder(std::move(video_encoder_config),
4820 VideoStreamEncoder::BitrateAllocationCallbackType::
4821 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004822
4823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004825
4826 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4827 WaitForEncodedFrame(CurrentTimeMs());
4828 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4829 VideoLayersAllocation last_layer_allocation =
4830 sink_.GetLastVideoLayersAllocation();
4831
4832 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4833 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4834 .target_bitrate_per_temporal_layer,
4835 SizeIs(2));
4836 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4837 .target_bitrate_per_temporal_layer,
4838 SizeIs(2));
4839 // Since KSVC is, spatial layers are independend except on key frames.
4840 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4841 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004842 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004843 video_stream_encoder_->Stop();
4844}
4845
4846TEST_F(VideoStreamEncoderTest,
4847 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4848 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4849 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4850 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851 VideoEncoderConfig video_encoder_config;
4852 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4853 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004854 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004855 video_encoder_config.content_type =
4856 VideoEncoderConfig::ContentType::kRealtimeVideo;
4857 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4858 vp9_settings.numberOfSpatialLayers = 3;
4859 vp9_settings.numberOfTemporalLayers = 2;
4860 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4861 vp9_settings.automaticResizeOn = false;
4862 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004863 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004864 vp9_settings);
4865 // Simulcast layers are used for enabling/disabling streams.
4866 video_encoder_config.simulcast_layers.resize(3);
4867 video_encoder_config.simulcast_layers[0].active = false;
4868 video_encoder_config.simulcast_layers[1].active = true;
4869 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004870 ConfigureEncoder(std::move(video_encoder_config),
4871 VideoStreamEncoder::BitrateAllocationCallbackType::
4872 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004873
4874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004875 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004876
4877 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4878 WaitForEncodedFrame(CurrentTimeMs());
4879 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4880 VideoLayersAllocation last_layer_allocation =
4881 sink_.GetLastVideoLayersAllocation();
4882
4883 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4884 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4885 .target_bitrate_per_temporal_layer,
4886 SizeIs(2));
4887 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4888 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4889
4890 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4891 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4892 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4893 .target_bitrate_per_temporal_layer,
4894 SizeIs(2));
4895 // Since full SVC is used, expect the top layer to utilize the full target
4896 // rate.
4897 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4898 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004899 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004900 video_stream_encoder_->Stop();
4901}
4902
4903TEST_F(VideoStreamEncoderTest,
4904 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4907 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004908 VideoEncoderConfig video_encoder_config;
4909 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4910 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004911 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004912 video_encoder_config.content_type =
4913 VideoEncoderConfig::ContentType::kRealtimeVideo;
4914 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4915 vp9_settings.numberOfSpatialLayers = 3;
4916 vp9_settings.numberOfTemporalLayers = 2;
4917 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4918 vp9_settings.automaticResizeOn = false;
4919 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004920 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004921 vp9_settings);
4922 // Simulcast layers are used for enabling/disabling streams.
4923 video_encoder_config.simulcast_layers.resize(3);
4924 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004925 ConfigureEncoder(std::move(video_encoder_config),
4926 VideoStreamEncoder::BitrateAllocationCallbackType::
4927 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004928
4929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004930 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004931
4932 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4933 WaitForEncodedFrame(CurrentTimeMs());
4934 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4935 VideoLayersAllocation last_layer_allocation =
4936 sink_.GetLastVideoLayersAllocation();
4937
4938 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4939 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4940 .target_bitrate_per_temporal_layer,
4941 SizeIs(2));
4942 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4943 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4944
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4946 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4947 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4948 .target_bitrate_per_temporal_layer,
4949 SizeIs(2));
4950 video_stream_encoder_->Stop();
4951}
4952
4953TEST_F(VideoStreamEncoderTest,
4954 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4955 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4956 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4957 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004958 VideoEncoderConfig video_encoder_config;
4959 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4960 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004961 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004962 video_encoder_config.content_type =
4963 VideoEncoderConfig::ContentType::kRealtimeVideo;
4964 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4965 vp9_settings.numberOfSpatialLayers = 3;
4966 vp9_settings.numberOfTemporalLayers = 2;
4967 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4968 vp9_settings.automaticResizeOn = false;
4969 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004970 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004971 vp9_settings);
4972 // Simulcast layers are used for enabling/disabling streams.
4973 video_encoder_config.simulcast_layers.resize(3);
4974 video_encoder_config.simulcast_layers[0].active = false;
4975 video_encoder_config.simulcast_layers[1].active = false;
4976 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004977 ConfigureEncoder(std::move(video_encoder_config),
4978 VideoStreamEncoder::BitrateAllocationCallbackType::
4979 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004980
4981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004983
4984 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4985 WaitForEncodedFrame(CurrentTimeMs());
4986 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4987 VideoLayersAllocation last_layer_allocation =
4988 sink_.GetLastVideoLayersAllocation();
4989
4990 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4991 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4992 .target_bitrate_per_temporal_layer,
4993 SizeIs(2));
4994 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4995 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4996 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4997 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004998 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004999 video_stream_encoder_->Stop();
5000}
5001
5002TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5003 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005004 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005005 kVideoLayersAllocation);
5006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005007 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005008
5009 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5010 WaitForEncodedFrame(CurrentTimeMs());
5011 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5012 VideoLayersAllocation last_layer_allocation =
5013 sink_.GetLastVideoLayersAllocation();
5014
5015 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5016 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5017 .target_bitrate_per_temporal_layer,
5018 SizeIs(1));
5019 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5020 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005021 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005022 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5023 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5024 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5025 video_stream_encoder_->Stop();
5026}
5027
5028TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005029 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5030 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005031 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005032 kVideoLayersAllocation);
5033
5034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005035 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005036
5037 video_source_.IncomingCapturedFrame(
5038 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5039 WaitForEncodedFrame(CurrentTimeMs());
5040 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5041 VideoLayersAllocation last_layer_allocation =
5042 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005043 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005044 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5045 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5046 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005047 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005048
5049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005050 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5051 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005052 video_source_.IncomingCapturedFrame(
5053 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5054 WaitForEncodedFrame(CurrentTimeMs());
5055
5056 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5057 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5058 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5059 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5060 .target_bitrate_per_temporal_layer[0],
5061 DataRate::Zero());
5062
5063 video_stream_encoder_->Stop();
5064}
5065
Per Kjellander4190ce92020-12-15 17:24:55 +01005066TEST_F(VideoStreamEncoderTest,
5067 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5068 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005069 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005070 kVideoLayersAllocation);
5071
5072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005073 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5074 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005075
5076 video_source_.IncomingCapturedFrame(
5077 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5078 WaitForEncodedFrame(CurrentTimeMs());
5079 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5080 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5081 SizeIs(2));
5082 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5083 codec_width_);
5084 EXPECT_EQ(
5085 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5086 codec_height_);
5087
5088 video_source_.IncomingCapturedFrame(
5089 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5090 WaitForEncodedFrame(CurrentTimeMs());
5091 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5092 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5093 SizeIs(2));
5094 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5095 codec_width_ / 2);
5096 EXPECT_EQ(
5097 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5098 codec_height_ / 2);
5099
5100 video_stream_encoder_->Stop();
5101}
5102
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005103TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5104 // 2 TLs configured, temporal layers supported by encoder.
5105 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005106 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005107 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005108 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005109 fake_encoder_.SetTemporalLayersSupported(0, true);
5110
5111 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005112 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005113 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005114 kNumTemporalLayers, /*temporal_id*/ 0,
5115 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005116 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005117 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005118 kNumTemporalLayers, /*temporal_id*/ 1,
5119 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005120 VideoBitrateAllocation expected_bitrate;
5121 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5122 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5123
5124 VerifyAllocatedBitrate(expected_bitrate);
5125 video_stream_encoder_->Stop();
5126}
5127
5128TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5129 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005130 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005131 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005132 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005133 fake_encoder_.SetTemporalLayersSupported(0, false);
5134
5135 // Temporal layers not supported by the encoder.
5136 // Total bitrate should be at ti:0.
5137 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005138 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005139
5140 VerifyAllocatedBitrate(expected_bitrate);
5141 video_stream_encoder_->Stop();
5142}
5143
5144TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005145 webrtc::test::ScopedFieldTrials field_trials(
5146 "WebRTC-Video-QualityScalerSettings/"
5147 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5148 // Reset encoder for field trials to take effect.
5149 ConfigureEncoder(video_encoder_config_.Copy());
5150
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005151 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005152 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005153 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005154 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005155 fake_encoder_.SetTemporalLayersSupported(0, true);
5156 fake_encoder_.SetTemporalLayersSupported(1, false);
5157
5158 const int kS0Bps = 150000;
5159 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005160 kS0Bps *
5161 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5162 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005163 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005164 kS0Bps *
5165 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5166 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005167 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005168 // Temporal layers not supported by si:1.
5169 VideoBitrateAllocation expected_bitrate;
5170 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5171 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5172 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5173
5174 VerifyAllocatedBitrate(expected_bitrate);
5175 video_stream_encoder_->Stop();
5176}
5177
Niels Möller7dc26b72017-12-06 10:27:48 +01005178TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5179 const int kFrameWidth = 1280;
5180 const int kFrameHeight = 720;
5181 const int kFramerate = 24;
5182
Henrik Boström381d1092020-05-12 18:49:07 +02005183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005184 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005185 test::FrameForwarder source;
5186 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005187 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005188
5189 // Insert a single frame, triggering initial configuration.
5190 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5191 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5192
5193 EXPECT_EQ(
5194 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5195 kDefaultFramerate);
5196
5197 // Trigger reconfigure encoder (without resetting the entire instance).
5198 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005199 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5200 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005201 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005202 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005203 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005204 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5205
5206 // Detector should be updated with fps limit from codec config.
5207 EXPECT_EQ(
5208 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5209 kFramerate);
5210
5211 // Trigger overuse, max framerate should be reduced.
5212 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5213 stats.input_frame_rate = kFramerate;
5214 stats_proxy_->SetMockStats(stats);
5215 video_stream_encoder_->TriggerCpuOveruse();
5216 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5217 int adapted_framerate =
5218 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5219 EXPECT_LT(adapted_framerate, kFramerate);
5220
5221 // Trigger underuse, max framerate should go back to codec configured fps.
5222 // Set extra low fps, to make sure it's actually reset, not just incremented.
5223 stats = stats_proxy_->GetStats();
5224 stats.input_frame_rate = adapted_framerate / 2;
5225 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005226 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005227 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5228 EXPECT_EQ(
5229 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5230 kFramerate);
5231
5232 video_stream_encoder_->Stop();
5233}
5234
5235TEST_F(VideoStreamEncoderTest,
5236 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5237 const int kFrameWidth = 1280;
5238 const int kFrameHeight = 720;
5239 const int kLowFramerate = 15;
5240 const int kHighFramerate = 25;
5241
Henrik Boström381d1092020-05-12 18:49:07 +02005242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005243 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005244 test::FrameForwarder source;
5245 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005246 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005247
5248 // Trigger initial configuration.
5249 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005250 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5251 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005252 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005253 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005254 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005255 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005256 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5257
5258 EXPECT_EQ(
5259 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5260 kLowFramerate);
5261
5262 // Trigger overuse, max framerate should be reduced.
5263 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5264 stats.input_frame_rate = kLowFramerate;
5265 stats_proxy_->SetMockStats(stats);
5266 video_stream_encoder_->TriggerCpuOveruse();
5267 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5268 int adapted_framerate =
5269 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5270 EXPECT_LT(adapted_framerate, kLowFramerate);
5271
5272 // Reconfigure the encoder with a new (higher max framerate), max fps should
5273 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005274 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005275 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5276 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005277 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005278 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5279
5280 EXPECT_EQ(
5281 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5282 adapted_framerate);
5283
5284 // Trigger underuse, max framerate should go back to codec configured fps.
5285 stats = stats_proxy_->GetStats();
5286 stats.input_frame_rate = adapted_framerate;
5287 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005288 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005289 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5290 EXPECT_EQ(
5291 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5292 kHighFramerate);
5293
5294 video_stream_encoder_->Stop();
5295}
5296
mflodmancc3d4422017-08-03 08:27:51 -07005297TEST_F(VideoStreamEncoderTest,
5298 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005299 const int kFrameWidth = 1280;
5300 const int kFrameHeight = 720;
5301 const int kFramerate = 24;
5302
Henrik Boström381d1092020-05-12 18:49:07 +02005303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005304 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005305 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005306 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005307 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005308
5309 // Trigger initial configuration.
5310 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005311 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5312 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005313 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005314 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005315 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005316 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005317 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005318
Niels Möller7dc26b72017-12-06 10:27:48 +01005319 EXPECT_EQ(
5320 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5321 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005322
5323 // Trigger overuse, max framerate should be reduced.
5324 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5325 stats.input_frame_rate = kFramerate;
5326 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005327 video_stream_encoder_->TriggerCpuOveruse();
5328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005329 int adapted_framerate =
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005331 EXPECT_LT(adapted_framerate, kFramerate);
5332
5333 // Change degradation preference to not enable framerate scaling. Target
5334 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005335 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005336 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005337 EXPECT_EQ(
5338 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5339 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005340
mflodmancc3d4422017-08-03 08:27:51 -07005341 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005342}
5343
mflodmancc3d4422017-08-03 08:27:51 -07005344TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005345 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005347 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5348 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5349 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005350 const int kWidth = 640;
5351 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005352
asaperssonfab67072017-04-04 05:51:49 -07005353 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005354
5355 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005356 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005357
5358 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005359 EXPECT_TRUE_WAIT(
5360 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005361
sprangc5d62e22017-04-02 23:53:04 -07005362 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005363
asaperssonfab67072017-04-04 05:51:49 -07005364 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005365 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005366 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005367
5368 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005369 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005370
Henrik Boström2671dac2020-05-19 16:29:09 +02005371 EXPECT_TRUE_WAIT(
5372 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005373
mflodmancc3d4422017-08-03 08:27:51 -07005374 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005375}
5376
mflodmancc3d4422017-08-03 08:27:51 -07005377TEST_F(VideoStreamEncoderTest,
5378 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005379 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005381 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5382 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5383 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005384 const int kWidth = 640;
5385 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005386
5387 // We expect the n initial frames to get dropped.
5388 int i;
5389 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005390 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005391 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005392 }
5393 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005394 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005395 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005396
5397 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005398 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005399
mflodmancc3d4422017-08-03 08:27:51 -07005400 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005401}
5402
mflodmancc3d4422017-08-03 08:27:51 -07005403TEST_F(VideoStreamEncoderTest,
5404 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005405 const int kWidth = 640;
5406 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005408 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005409
5410 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005411 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005412 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005413
asaperssonfab67072017-04-04 05:51:49 -07005414 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005415 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005416 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005417
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005419}
5420
mflodmancc3d4422017-08-03 08:27:51 -07005421TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005422 const int kWidth = 640;
5423 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005424 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005425
5426 VideoEncoderConfig video_encoder_config;
5427 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5428 // Make format different, to force recreation of encoder.
5429 video_encoder_config.video_format.parameters["foo"] = "foo";
5430 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005431 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005433 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005434
kthelgasonb83797b2017-02-14 11:57:25 -08005435 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005436 video_stream_encoder_->SetSource(&video_source_,
5437 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005438
asaperssonfab67072017-04-04 05:51:49 -07005439 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005440 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005441 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005442
mflodmancc3d4422017-08-03 08:27:51 -07005443 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005444 fake_encoder_.SetQualityScaling(true);
5445}
5446
Åsa Persson139f4dc2019-08-02 09:29:58 +02005447TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5448 webrtc::test::ScopedFieldTrials field_trials(
5449 "WebRTC-Video-QualityScalerSettings/"
5450 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5451 // Reset encoder for field trials to take effect.
5452 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005453 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5454 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005455 const int kWidth = 640;
5456 const int kHeight = 360;
5457
Henrik Boström381d1092020-05-12 18:49:07 +02005458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005459 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005460 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5461 // Frame should not be dropped.
5462 WaitForEncodedFrame(1);
5463
Henrik Boström381d1092020-05-12 18:49:07 +02005464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005465 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5466 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5467 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005468 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5469 // Frame should not be dropped.
5470 WaitForEncodedFrame(2);
5471
Henrik Boström381d1092020-05-12 18:49:07 +02005472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005473 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5474 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5475 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005476 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5477 // Expect to drop this frame, the wait should time out.
5478 ExpectDroppedFrame();
5479
5480 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005481 EXPECT_TRUE_WAIT(
5482 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005483 video_stream_encoder_->Stop();
5484}
5485
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005486TEST_F(VideoStreamEncoderTest,
5487 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5488 webrtc::test::ScopedFieldTrials field_trials(
5489 "WebRTC-Video-QualityScalerSettings/"
5490 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5491 fake_encoder_.SetQualityScaling(false);
5492 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005493 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5494 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005495 const int kWidth = 640;
5496 const int kHeight = 360;
5497
5498 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005499 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005500 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5501 // Frame should not be dropped.
5502 WaitForEncodedFrame(1);
5503
5504 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5505 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5506 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5507 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5508 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5509 // Frame should not be dropped.
5510 WaitForEncodedFrame(2);
5511
5512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5513 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5514 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5515 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5516 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5517 // Not dropped since quality scaling is disabled.
5518 WaitForEncodedFrame(3);
5519
5520 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005521 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005522 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5523
5524 video_stream_encoder_->Stop();
5525}
5526
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005527TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005528 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005529 // Set simulcast.
5530 ResetEncoder("VP8", 3, 1, 1, false);
5531 fake_encoder_.SetQualityScaling(true);
5532 const int kWidth = 1280;
5533 const int kHeight = 720;
5534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005535 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005536 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5537 // Frame should not be dropped.
5538 WaitForEncodedFrame(1);
5539
5540 // Trigger QVGA "singlecast"
5541 // Update the config.
5542 VideoEncoderConfig video_encoder_config;
5543 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5544 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005545 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005546 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005547 "VP8", /*max qp*/ 56, /*screencast*/ false,
5548 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005549 for (auto& layer : video_encoder_config.simulcast_layers) {
5550 layer.num_temporal_layers = 1;
5551 layer.max_framerate = kDefaultFramerate;
5552 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005553 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005554 video_encoder_config.content_type =
5555 VideoEncoderConfig::ContentType::kRealtimeVideo;
5556
5557 video_encoder_config.simulcast_layers[0].active = true;
5558 video_encoder_config.simulcast_layers[1].active = false;
5559 video_encoder_config.simulcast_layers[2].active = false;
5560
5561 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5562 kMaxPayloadLength);
5563 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5564
5565 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5566 // Frame should not be dropped.
5567 WaitForEncodedFrame(2);
5568
5569 // Trigger HD "singlecast"
5570 video_encoder_config.simulcast_layers[0].active = false;
5571 video_encoder_config.simulcast_layers[1].active = false;
5572 video_encoder_config.simulcast_layers[2].active = true;
5573
5574 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5575 kMaxPayloadLength);
5576 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5577
5578 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5579 // Frame should be dropped because of initial frame drop.
5580 ExpectDroppedFrame();
5581
5582 // Expect the sink_wants to specify a scaled frame.
5583 EXPECT_TRUE_WAIT(
5584 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5585 video_stream_encoder_->Stop();
5586}
5587
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005588TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005589 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005590 // Set simulcast.
5591 ResetEncoder("VP9", 1, 1, 3, false);
5592 fake_encoder_.SetQualityScaling(true);
5593 const int kWidth = 1280;
5594 const int kHeight = 720;
5595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005596 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005597 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5598 // Frame should not be dropped.
5599 WaitForEncodedFrame(1);
5600
5601 // Trigger QVGA "singlecast"
5602 // Update the config.
5603 VideoEncoderConfig video_encoder_config;
5604 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5605 &video_encoder_config);
5606 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5607 vp9_settings.numberOfSpatialLayers = 3;
5608 // Since only one layer is active - automatic resize should be enabled.
5609 vp9_settings.automaticResizeOn = true;
5610 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005611 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005612 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005613 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005614 video_encoder_config.content_type =
5615 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005616 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005617 // which SVC layers are active.
5618 video_encoder_config.simulcast_layers.resize(3);
5619
5620 video_encoder_config.simulcast_layers[0].active = true;
5621 video_encoder_config.simulcast_layers[1].active = false;
5622 video_encoder_config.simulcast_layers[2].active = false;
5623
5624 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5625 kMaxPayloadLength);
5626 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5627
5628 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5629 // Frame should not be dropped.
5630 WaitForEncodedFrame(2);
5631
5632 // Trigger HD "singlecast"
5633 video_encoder_config.simulcast_layers[0].active = false;
5634 video_encoder_config.simulcast_layers[1].active = false;
5635 video_encoder_config.simulcast_layers[2].active = true;
5636
5637 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5638 kMaxPayloadLength);
5639 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5640
5641 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5642 // Frame should be dropped because of initial frame drop.
5643 ExpectDroppedFrame();
5644
5645 // Expect the sink_wants to specify a scaled frame.
5646 EXPECT_TRUE_WAIT(
5647 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5648 video_stream_encoder_->Stop();
5649}
5650
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005651TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005652 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5653 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5654 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5655 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5656 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5657 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5658 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5659 fake_encoder_.SetResolutionBitrateLimits(
5660 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5661
5662 VideoEncoderConfig video_encoder_config;
5663 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5664 &video_encoder_config);
5665 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5666 vp9_settings.numberOfSpatialLayers = 3;
5667 // Since only one layer is active - automatic resize should be enabled.
5668 vp9_settings.automaticResizeOn = true;
5669 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005670 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005671 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005672 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005673 video_encoder_config.content_type =
5674 VideoEncoderConfig::ContentType::kRealtimeVideo;
5675 // Simulcast layers are used to indicate which spatial layers are active.
5676 video_encoder_config.simulcast_layers.resize(3);
5677 video_encoder_config.simulcast_layers[0].active = false;
5678 video_encoder_config.simulcast_layers[1].active = true;
5679 video_encoder_config.simulcast_layers[2].active = false;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 // The encoder bitrate limits for 360p should be used.
5686 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5687 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005688 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5689 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5690 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5691 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5692 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5693 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005694 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005695 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005696 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005697 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005698
5699 // The encoder bitrate limits for 270p should be used.
5700 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5701 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005702 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5703 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5704 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5705 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5706 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5707 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005708 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005709 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005710 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005711 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005712
5713 video_stream_encoder_->Stop();
5714}
5715
5716TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005717 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5718 VideoEncoderConfig video_encoder_config;
5719 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5720 &video_encoder_config);
5721 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5722 vp9_settings.numberOfSpatialLayers = 3;
5723 // Since only one layer is active - automatic resize should be enabled.
5724 vp9_settings.automaticResizeOn = true;
5725 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005726 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005727 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005728 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005729 video_encoder_config.content_type =
5730 VideoEncoderConfig::ContentType::kRealtimeVideo;
5731 // Simulcast layers are used to indicate which spatial layers are active.
5732 video_encoder_config.simulcast_layers.resize(3);
5733 video_encoder_config.simulcast_layers[0].active = false;
5734 video_encoder_config.simulcast_layers[1].active = true;
5735 video_encoder_config.simulcast_layers[2].active = false;
5736
5737 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5738 kMaxPayloadLength);
5739 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5740
5741 // The default bitrate limits for 360p should be used.
5742 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005743 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5744 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005745 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5746 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005747 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5748 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5749 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5750 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5751 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5752 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005753 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005754 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005755 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005756 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005757
5758 // The default bitrate limits for 270p should be used.
5759 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005760 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5761 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005762 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5763 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005764 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5765 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5766 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5767 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5768 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5769 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005770 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005771 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005772 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005773 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005774
5775 video_stream_encoder_->Stop();
5776}
5777
5778TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5779 webrtc::test::ScopedFieldTrials field_trials(
5780 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5781 VideoEncoderConfig video_encoder_config;
5782 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5783 &video_encoder_config);
5784 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5785 vp9_settings.numberOfSpatialLayers = 3;
5786 // Since only one layer is active - automatic resize should be enabled.
5787 vp9_settings.automaticResizeOn = true;
5788 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005789 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005790 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005791 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005792 video_encoder_config.content_type =
5793 VideoEncoderConfig::ContentType::kRealtimeVideo;
5794 // Simulcast layers are used to indicate which spatial layers are active.
5795 video_encoder_config.simulcast_layers.resize(3);
5796 video_encoder_config.simulcast_layers[0].active = false;
5797 video_encoder_config.simulcast_layers[1].active = true;
5798 video_encoder_config.simulcast_layers[2].active = false;
5799
5800 // Reset encoder for field trials to take effect.
5801 ConfigureEncoder(video_encoder_config.Copy());
5802
5803 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5804 kMaxPayloadLength);
5805 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5806
5807 // The default bitrate limits for 360p should not be used.
5808 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005809 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5810 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005811 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5812 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005813 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5814 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5815 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5816 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5817 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5818 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005819 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005820 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005821
5822 video_stream_encoder_->Stop();
5823}
5824
5825TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5826 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5827 /*num_spatial_layers=*/1, /*screenshare=*/false);
5828
5829 // The default singlecast bitrate limits for 720p should not be used.
5830 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005831 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5832 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005833 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5834 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005835 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5836 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5837 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5838 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5839 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5840 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005841 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005842 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005843
5844 video_stream_encoder_->Stop();
5845}
5846
5847TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005848 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5849 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5850 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5851 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5852 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5853 fake_encoder_.SetResolutionBitrateLimits(
5854 {kEncoderLimits180p, kEncoderLimits720p});
5855
5856 VideoEncoderConfig video_encoder_config;
5857 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5858 &video_encoder_config);
5859 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5860 vp9_settings.numberOfSpatialLayers = 3;
5861 // Since only one layer is active - automatic resize should be enabled.
5862 vp9_settings.automaticResizeOn = true;
5863 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005864 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005865 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005866 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005867 video_encoder_config.content_type =
5868 VideoEncoderConfig::ContentType::kRealtimeVideo;
5869 // Simulcast layers are used to indicate which spatial layers are active.
5870 video_encoder_config.simulcast_layers.resize(3);
5871 video_encoder_config.simulcast_layers[0].active = true;
5872 video_encoder_config.simulcast_layers[1].active = false;
5873 video_encoder_config.simulcast_layers[2].active = false;
5874
5875 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5876 kMaxPayloadLength);
5877 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5878
5879 // Limits not applied on lowest stream, limits for 180p should not be used.
5880 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5881 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005882 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5883 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5884 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5885 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5886 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5887 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005888 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005889 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005890 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005891 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005892
5893 video_stream_encoder_->Stop();
5894}
5895
5896TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005897 InitialFrameDropActivatesWhenResolutionIncreases) {
5898 const int kWidth = 640;
5899 const int kHeight = 360;
5900
5901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005902 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005903 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5904 // Frame should not be dropped.
5905 WaitForEncodedFrame(1);
5906
5907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005908 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005909 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5910 // Frame should not be dropped, bitrate not too low for frame.
5911 WaitForEncodedFrame(2);
5912
5913 // Incoming resolution increases.
5914 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5915 // Expect to drop this frame, bitrate too low for frame.
5916 ExpectDroppedFrame();
5917
5918 // Expect the sink_wants to specify a scaled frame.
5919 EXPECT_TRUE_WAIT(
5920 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5921 video_stream_encoder_->Stop();
5922}
5923
5924TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5925 const int kWidth = 640;
5926 const int kHeight = 360;
5927 // So that quality scaling doesn't happen by itself.
5928 fake_encoder_.SetQp(kQpHigh);
5929
5930 AdaptingFrameForwarder source(&time_controller_);
5931 source.set_adaptation_enabled(true);
5932 video_stream_encoder_->SetSource(
5933 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5934
5935 int timestamp = 1;
5936
5937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005938 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005939 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5940 WaitForEncodedFrame(timestamp);
5941 timestamp += 9000;
5942 // Long pause to disable all first BWE drop logic.
5943 AdvanceTime(TimeDelta::Millis(1000));
5944
5945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005946 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005947 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5948 // Not dropped frame, as initial frame drop is disabled by now.
5949 WaitForEncodedFrame(timestamp);
5950 timestamp += 9000;
5951 AdvanceTime(TimeDelta::Millis(100));
5952
5953 // Quality adaptation down.
5954 video_stream_encoder_->TriggerQualityLow();
5955
5956 // Adaptation has an effect.
5957 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5958 5000);
5959
5960 // Frame isn't dropped as initial frame dropper is disabled.
5961 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5962 WaitForEncodedFrame(timestamp);
5963 timestamp += 9000;
5964 AdvanceTime(TimeDelta::Millis(100));
5965
5966 // Quality adaptation up.
5967 video_stream_encoder_->TriggerQualityHigh();
5968
5969 // Adaptation has an effect.
5970 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5971 5000);
5972
5973 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5974 // Frame should not be dropped, as initial framedropper is off.
5975 WaitForEncodedFrame(timestamp);
5976
5977 video_stream_encoder_->Stop();
5978}
5979
Åsa Persson7f354f82021-02-04 15:52:15 +01005980TEST_F(VideoStreamEncoderTest,
5981 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5982 const int kMinStartBps360p = 222000;
5983 fake_encoder_.SetResolutionBitrateLimits(
5984 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5985 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5986 800000)});
5987
5988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5989 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5990 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5991 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5992 0, 0, 0);
5993 // Frame should not be dropped, bitrate not too low for frame.
5994 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5995 WaitForEncodedFrame(1);
5996
5997 // Incoming resolution increases, initial frame drop activates.
5998 // Frame should be dropped, link allocation too low for frame.
5999 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6000 ExpectDroppedFrame();
6001
6002 // Expect sink_wants to specify a scaled frame.
6003 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6004 5000);
6005 video_stream_encoder_->Stop();
6006}
6007
6008TEST_F(VideoStreamEncoderTest,
6009 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6010 const int kMinStartBps360p = 222000;
6011 fake_encoder_.SetResolutionBitrateLimits(
6012 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6013 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6014 800000)});
6015
6016 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6017 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6018 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6019 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6020 0, 0, 0);
6021 // Frame should not be dropped, bitrate not too low for frame.
6022 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6023 WaitForEncodedFrame(1);
6024
6025 // Incoming resolution increases, initial frame drop activates.
6026 // Frame should be dropped, link allocation not too low for frame.
6027 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6028 WaitForEncodedFrame(2);
6029
6030 video_stream_encoder_->Stop();
6031}
6032
Åsa Perssone644a032019-11-08 15:56:00 +01006033TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6034 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006035 "WebRTC-Video-QualityRampupSettings/"
6036 "min_pixels:921600,min_duration_ms:2000/");
6037
6038 const int kWidth = 1280;
6039 const int kHeight = 720;
6040 const int kFps = 10;
6041 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006042
6043 // Reset encoder for field trials to take effect.
6044 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006045 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006046 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006047 ConfigureEncoder(std::move(config));
6048 fake_encoder_.SetQp(kQpLow);
6049
6050 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006051 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006052 source.set_adaptation_enabled(true);
6053 video_stream_encoder_->SetSource(&source,
6054 DegradationPreference::MAINTAIN_FRAMERATE);
6055
6056 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006057 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006059 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006060
6061 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006062 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006063 int64_t timestamp_ms = kFrameIntervalMs;
6064 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6065 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006066 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6067 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006068
6069 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6071 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006072
Artem Titovab30d722021-07-27 16:22:11 +02006073 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006074 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006075 for (size_t i = 1; i <= 10; i++) {
6076 timestamp_ms += kFrameIntervalMs;
6077 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6078 WaitForEncodedFrame(timestamp_ms);
6079 }
Åsa Persson06defc42021-09-10 15:28:48 +02006080
6081 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6082 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6083 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6084 timestamp_ms += kFrameIntervalMs;
6085 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6086 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006087 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6088 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6089
Åsa Persson06defc42021-09-10 15:28:48 +02006090 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006091 timestamp_ms += kFrameIntervalMs;
6092 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6093 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006094 // The ramp-up code involves the adaptation queue, give it time to execute.
6095 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006096 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006097 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006098
6099 // Frame should not be adapted.
6100 timestamp_ms += kFrameIntervalMs;
6101 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6102 WaitForEncodedFrame(kWidth, kHeight);
6103 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6104
6105 video_stream_encoder_->Stop();
6106}
6107
mflodmancc3d4422017-08-03 08:27:51 -07006108TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006109 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006110 webrtc::test::ScopedFieldTrials field_trials(
6111 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006112 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006113 source.set_adaptation_enabled(true);
6114 video_stream_encoder_->SetSource(&source,
6115 DegradationPreference::MAINTAIN_FRAMERATE);
6116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006117 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006118 fake_encoder_.SetQp(kQpHigh + 1);
6119 const int kWidth = 1280;
6120 const int kHeight = 720;
6121 const int64_t kFrameIntervalMs = 100;
6122 int64_t timestamp_ms = kFrameIntervalMs;
6123 for (size_t i = 1; i <= 100; i++) {
6124 timestamp_ms += kFrameIntervalMs;
6125 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6126 WaitForEncodedFrame(timestamp_ms);
6127 }
6128 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6129 // for the first time.
6130 // TODO(eshr): We should avoid these waits by using threads with simulated
6131 // time.
6132 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6133 2000 * 2.5 * 2);
6134 timestamp_ms += kFrameIntervalMs;
6135 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6136 WaitForEncodedFrame(timestamp_ms);
6137 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6138 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6139 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6140
6141 // Disable Quality scaling by turning off scaler on the encoder and
6142 // reconfiguring.
6143 fake_encoder_.SetQualityScaling(false);
6144 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6145 kMaxPayloadLength);
6146 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006147 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006148 // Since we turned off the quality scaler, the adaptations made by it are
6149 // removed.
6150 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6151 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6152
6153 video_stream_encoder_->Stop();
6154}
6155
6156TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006157 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6158 const int kTooSmallWidth = 10;
6159 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006162
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006163 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006164 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006165 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006166 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006167 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006168 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6169
6170 // Trigger adapt down, too small frame, expect no change.
6171 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006172 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006173 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006174 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006175 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6176 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6177
mflodmancc3d4422017-08-03 08:27:51 -07006178 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006179}
6180
mflodmancc3d4422017-08-03 08:27:51 -07006181TEST_F(VideoStreamEncoderTest,
6182 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006183 const int kTooSmallWidth = 10;
6184 const int kTooSmallHeight = 10;
6185 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006186 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006187 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006188
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006189 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006190 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006191 video_stream_encoder_->SetSource(&source,
6192 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006193 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006194 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6195 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6196
6197 // Trigger adapt down, expect limited framerate.
6198 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006199 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006200 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006201 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006202 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6203 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6204 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6205
6206 // Trigger adapt down, too small frame, expect no change.
6207 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006208 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006209 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006210 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006211 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6212 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6213 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6214
mflodmancc3d4422017-08-03 08:27:51 -07006215 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006216}
6217
mflodmancc3d4422017-08-03 08:27:51 -07006218TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006219 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006221 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006222 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006223 const int kFrameWidth = 1280;
6224 const int kFrameHeight = 720;
6225 video_source_.IncomingCapturedFrame(
6226 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006227 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006228 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006229}
6230
sprangb1ca0732017-02-01 08:38:12 -08006231// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006232TEST_F(VideoStreamEncoderTest,
6233 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006234 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006235 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006236
6237 const int kFrameWidth = 1280;
6238 const int kFrameHeight = 720;
6239 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006240 // requested by
6241 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006242 video_source_.set_adaptation_enabled(true);
6243
6244 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006245 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006246 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006247
6248 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006249 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006250 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006251 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006252 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006253
asaperssonfab67072017-04-04 05:51:49 -07006254 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006255 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006256 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006257 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006258 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006259
mflodmancc3d4422017-08-03 08:27:51 -07006260 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006261}
sprangfe627f32017-03-29 08:24:59 -07006262
mflodmancc3d4422017-08-03 08:27:51 -07006263TEST_F(VideoStreamEncoderTest,
6264 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006265 const int kFrameWidth = 1280;
6266 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006267
Henrik Boström381d1092020-05-12 18:49:07 +02006268 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006269 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006270 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006271 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006272 video_source_.set_adaptation_enabled(true);
6273
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006274 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006275
6276 video_source_.IncomingCapturedFrame(
6277 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006278 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006279
6280 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006281 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006282
6283 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006284 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006285 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006286 video_source_.IncomingCapturedFrame(
6287 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006288 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006289 }
6290
6291 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006292 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006293 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006294 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006295 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006296 video_source_.IncomingCapturedFrame(
6297 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006298 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006299 ++num_frames_dropped;
6300 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006301 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006302 }
6303 }
6304
sprang4847ae62017-06-27 07:06:52 -07006305 // Add some slack to account for frames dropped by the frame dropper.
6306 const int kErrorMargin = 1;
6307 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006308 kErrorMargin);
6309
6310 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006312 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006313 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006314 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006315 video_source_.IncomingCapturedFrame(
6316 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006317 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006318 ++num_frames_dropped;
6319 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006320 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006321 }
6322 }
sprang4847ae62017-06-27 07:06:52 -07006323 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006324 kErrorMargin);
6325
6326 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006327 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006328 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006329 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006330 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006331 video_source_.IncomingCapturedFrame(
6332 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006333 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006334 ++num_frames_dropped;
6335 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006336 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006337 }
6338 }
sprang4847ae62017-06-27 07:06:52 -07006339 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006340 kErrorMargin);
6341
6342 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006343 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006344 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006345 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006346 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006347 video_source_.IncomingCapturedFrame(
6348 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006349 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006350 ++num_frames_dropped;
6351 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006352 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006353 }
6354 }
6355 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6356
mflodmancc3d4422017-08-03 08:27:51 -07006357 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006358}
6359
mflodmancc3d4422017-08-03 08:27:51 -07006360TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006361 const int kFramerateFps = 5;
6362 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006363 const int kFrameWidth = 1280;
6364 const int kFrameHeight = 720;
6365
sprang4847ae62017-06-27 07:06:52 -07006366 // Reconfigure encoder with two temporal layers and screensharing, which will
6367 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006368 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006369
Henrik Boström381d1092020-05-12 18:49:07 +02006370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006371 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006373 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006374 video_source_.set_adaptation_enabled(true);
6375
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006376 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006377
6378 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006379 rtc::VideoSinkWants last_wants;
6380 do {
6381 last_wants = video_source_.sink_wants();
6382
sprangc5d62e22017-04-02 23:53:04 -07006383 // Insert frames to get a new fps estimate...
6384 for (int j = 0; j < kFramerateFps; ++j) {
6385 video_source_.IncomingCapturedFrame(
6386 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006387 if (video_source_.last_sent_width()) {
6388 sink_.WaitForEncodedFrame(timestamp_ms);
6389 }
sprangc5d62e22017-04-02 23:53:04 -07006390 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006391 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006392 }
6393 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006394 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006395 } while (video_source_.sink_wants().max_framerate_fps <
6396 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006397
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006398 EXPECT_THAT(video_source_.sink_wants(),
6399 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006400
mflodmancc3d4422017-08-03 08:27:51 -07006401 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006402}
asaperssonf7e294d2017-06-13 23:25:22 -07006403
mflodmancc3d4422017-08-03 08:27:51 -07006404TEST_F(VideoStreamEncoderTest,
6405 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006406 const int kWidth = 1280;
6407 const int kHeight = 720;
6408 const int64_t kFrameIntervalMs = 150;
6409 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006411 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006412
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006413 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006414 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006415 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006416 video_stream_encoder_->SetSource(&source,
6417 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006418 timestamp_ms += kFrameIntervalMs;
6419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006420 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006421 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006422 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6424 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6425
6426 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006427 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006428 timestamp_ms += kFrameIntervalMs;
6429 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006430 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006431 EXPECT_THAT(source.sink_wants(),
6432 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6435 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6436
6437 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006438 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006439 timestamp_ms += kFrameIntervalMs;
6440 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006441 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006442 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6444 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6445 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6446
6447 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006448 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006449 timestamp_ms += kFrameIntervalMs;
6450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006452 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6455 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6456
6457 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006458 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006459 timestamp_ms += kFrameIntervalMs;
6460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006461 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006462 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6465 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6466
6467 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006468 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006469 timestamp_ms += kFrameIntervalMs;
6470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006471 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006472 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006473 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6474 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6475 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6476
6477 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006478 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006479 timestamp_ms += kFrameIntervalMs;
6480 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006481 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006482 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6484 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6485 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6486
6487 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006488 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006489 timestamp_ms += kFrameIntervalMs;
6490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006491 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006492 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006493 rtc::VideoSinkWants last_wants = source.sink_wants();
6494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6496 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6497
6498 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006499 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006500 timestamp_ms += kFrameIntervalMs;
6501 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006502 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006503 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6506 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6507
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006508 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006509 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006510 timestamp_ms += kFrameIntervalMs;
6511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006512 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006513 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6516 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6517
6518 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006519 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006520 timestamp_ms += kFrameIntervalMs;
6521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006523 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6526 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6527
6528 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006533 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6536 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6537
6538 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006539 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006540 timestamp_ms += kFrameIntervalMs;
6541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006543 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6546 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6547
6548 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006549 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006550 timestamp_ms += kFrameIntervalMs;
6551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006553 EXPECT_THAT(source.sink_wants(), FpsMax());
6554 EXPECT_EQ(source.sink_wants().max_pixel_count,
6555 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6569
Åsa Persson30ab0152019-08-27 12:22:33 +02006570 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006571 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006572 timestamp_ms += kFrameIntervalMs;
6573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006574 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006575 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006576 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006577 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6579 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6580
6581 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006582 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006583 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006584 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6585
mflodmancc3d4422017-08-03 08:27:51 -07006586 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006587}
6588
mflodmancc3d4422017-08-03 08:27:51 -07006589TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006590 const int kWidth = 1280;
6591 const int kHeight = 720;
6592 const int64_t kFrameIntervalMs = 150;
6593 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006594 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006595 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006596
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006597 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006598 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006599 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006600 video_stream_encoder_->SetSource(&source,
6601 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006605 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6609 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6610 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6611 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6612
6613 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006614 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006615 timestamp_ms += kFrameIntervalMs;
6616 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006617 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006618 EXPECT_THAT(source.sink_wants(),
6619 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006620 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6623 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6624 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6625 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6626
6627 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006628 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006631 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006632 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006633 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6635 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6636 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6637 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6638 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
6640 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006644 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006645 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6648 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6650 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6651 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6652
Evan Shrubsole64469032020-06-11 10:45:29 +02006653 // Trigger cpu adapt up, expect no change since QP is most limited.
6654 {
6655 // Store current sink wants since we expect no change and if there is no
6656 // change then last_wants() is not updated.
6657 auto previous_sink_wants = source.sink_wants();
6658 video_stream_encoder_->TriggerCpuUnderuse();
6659 timestamp_ms += kFrameIntervalMs;
6660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6661 WaitForEncodedFrame(timestamp_ms);
6662 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6663 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6664 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6665 }
6666
6667 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6668 video_stream_encoder_->TriggerQualityHigh();
6669 timestamp_ms += kFrameIntervalMs;
6670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6671 WaitForEncodedFrame(timestamp_ms);
6672 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6673 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6675 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6677 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6678 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6679
6680 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6681 // expect increased resolution (960x540@30fps).
6682 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006683 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006684 timestamp_ms += kFrameIntervalMs;
6685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006686 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006687 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006688 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6690 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6692 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006693 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006694
Evan Shrubsole64469032020-06-11 10:45:29 +02006695 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6696 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006697 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006698 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006699 timestamp_ms += kFrameIntervalMs;
6700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006701 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006702 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006703 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006704 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6705 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6707 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6708 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006709 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006710
6711 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006712 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006713 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006714 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006715 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006716
mflodmancc3d4422017-08-03 08:27:51 -07006717 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006718}
6719
mflodmancc3d4422017-08-03 08:27:51 -07006720TEST_F(VideoStreamEncoderTest,
6721 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006722 const int kWidth = 640;
6723 const int kHeight = 360;
6724 const int kFpsLimit = 15;
6725 const int64_t kFrameIntervalMs = 150;
6726 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006728 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006729
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006730 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006731 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006732 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006733 video_stream_encoder_->SetSource(&source,
6734 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006735 timestamp_ms += kFrameIntervalMs;
6736 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006737 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006738 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006739 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6740 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6741 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6742 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6743 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6744 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6745
6746 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006747 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006748 timestamp_ms += kFrameIntervalMs;
6749 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006750 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006751 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006752 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6753 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6754 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6755 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6756 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6757 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6758
6759 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006760 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006761 timestamp_ms += kFrameIntervalMs;
6762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006763 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006764 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006765 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006766 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006767 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6768 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6769 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6770 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6771
Evan Shrubsole64469032020-06-11 10:45:29 +02006772 // Trigger cpu adapt up, expect no change because quality is most limited.
6773 {
6774 auto previous_sink_wants = source.sink_wants();
6775 // Store current sink wants since we expect no change ind if there is no
6776 // change then last__wants() is not updated.
6777 video_stream_encoder_->TriggerCpuUnderuse();
6778 timestamp_ms += kFrameIntervalMs;
6779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6780 WaitForEncodedFrame(timestamp_ms);
6781 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6782 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6783 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6784 }
6785
6786 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6787 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006788 timestamp_ms += kFrameIntervalMs;
6789 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006790 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006791 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6793 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6794 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006795 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6796 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6797 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006798
Evan Shrubsole64469032020-06-11 10:45:29 +02006799 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006800 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006801 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006802 timestamp_ms += kFrameIntervalMs;
6803 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006804 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6810 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006811 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006812
6813 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006814 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006816 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006817 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006818
mflodmancc3d4422017-08-03 08:27:51 -07006819 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006820}
6821
mflodmancc3d4422017-08-03 08:27:51 -07006822TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006823 const int kFrameWidth = 1920;
6824 const int kFrameHeight = 1080;
6825 // 3/4 of 1920.
6826 const int kAdaptedFrameWidth = 1440;
6827 // 3/4 of 1080 rounded down to multiple of 4.
6828 const int kAdaptedFrameHeight = 808;
6829 const int kFramerate = 24;
6830
Henrik Boström381d1092020-05-12 18:49:07 +02006831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006833 // Trigger reconfigure encoder (without resetting the entire instance).
6834 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006835 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6836 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006837 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006838 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006839 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006840 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006841 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006842 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006843
6844 video_source_.set_adaptation_enabled(true);
6845
6846 video_source_.IncomingCapturedFrame(
6847 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006848 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006849
6850 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006851 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006852 video_source_.IncomingCapturedFrame(
6853 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006854 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006855
mflodmancc3d4422017-08-03 08:27:51 -07006856 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006857}
6858
mflodmancc3d4422017-08-03 08:27:51 -07006859TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006860 const int kFrameWidth = 1280;
6861 const int kFrameHeight = 720;
6862 const int kLowFps = 2;
6863 const int kHighFps = 30;
6864
Henrik Boström381d1092020-05-12 18:49:07 +02006865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006867
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006868 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006869 max_framerate_ = kLowFps;
6870
6871 // Insert 2 seconds of 2fps video.
6872 for (int i = 0; i < kLowFps * 2; ++i) {
6873 video_source_.IncomingCapturedFrame(
6874 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6875 WaitForEncodedFrame(timestamp_ms);
6876 timestamp_ms += 1000 / kLowFps;
6877 }
6878
6879 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006881 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006882 video_source_.IncomingCapturedFrame(
6883 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6884 WaitForEncodedFrame(timestamp_ms);
6885 timestamp_ms += 1000 / kLowFps;
6886
6887 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6888
6889 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006890 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006891 const int kFrameIntervalMs = 1000 / kHighFps;
6892 max_framerate_ = kHighFps;
6893 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6894 video_source_.IncomingCapturedFrame(
6895 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6896 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6897 // be dropped if the encoder hans't been updated with the new higher target
6898 // framerate yet, causing it to overshoot the target bitrate and then
6899 // suffering the wrath of the media optimizer.
6900 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6901 timestamp_ms += kFrameIntervalMs;
6902 }
6903
6904 // Don expect correct measurement just yet, but it should be higher than
6905 // before.
6906 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6907
mflodmancc3d4422017-08-03 08:27:51 -07006908 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006909}
6910
mflodmancc3d4422017-08-03 08:27:51 -07006911TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006912 const int kFrameWidth = 1280;
6913 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006914 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006915 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006916 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006917
Henrik Boström381d1092020-05-12 18:49:07 +02006918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006919 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006920 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006921
6922 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006923 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006924 video_source_.IncomingCapturedFrame(
6925 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6926 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006927 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006928
6929 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006931 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006932
6933 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006934 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006935 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006936
Per Kjellanderdcef6412020-10-07 15:09:05 +02006937 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006938 video_source_.IncomingCapturedFrame(
6939 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6940 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006941 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006942
mflodmancc3d4422017-08-03 08:27:51 -07006943 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006944}
ilnik6b826ef2017-06-16 06:53:48 -07006945
Niels Möller4db138e2018-04-19 09:04:13 +02006946TEST_F(VideoStreamEncoderTest,
6947 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6948 const int kFrameWidth = 1280;
6949 const int kFrameHeight = 720;
6950 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006951 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006952 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006953 video_source_.IncomingCapturedFrame(
6954 CreateFrame(1, kFrameWidth, kFrameHeight));
6955 WaitForEncodedFrame(1);
6956 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6957 .low_encode_usage_threshold_percent,
6958 default_options.low_encode_usage_threshold_percent);
6959 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6960 .high_encode_usage_threshold_percent,
6961 default_options.high_encode_usage_threshold_percent);
6962 video_stream_encoder_->Stop();
6963}
6964
6965TEST_F(VideoStreamEncoderTest,
6966 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6967 const int kFrameWidth = 1280;
6968 const int kFrameHeight = 720;
6969 CpuOveruseOptions hardware_options;
6970 hardware_options.low_encode_usage_threshold_percent = 150;
6971 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006972 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006973
Henrik Boström381d1092020-05-12 18:49:07 +02006974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006975 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006976 video_source_.IncomingCapturedFrame(
6977 CreateFrame(1, kFrameWidth, kFrameHeight));
6978 WaitForEncodedFrame(1);
6979 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6980 .low_encode_usage_threshold_percent,
6981 hardware_options.low_encode_usage_threshold_percent);
6982 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6983 .high_encode_usage_threshold_percent,
6984 hardware_options.high_encode_usage_threshold_percent);
6985 video_stream_encoder_->Stop();
6986}
6987
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006988TEST_F(VideoStreamEncoderTest,
6989 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6990 const int kFrameWidth = 1280;
6991 const int kFrameHeight = 720;
6992
6993 const CpuOveruseOptions default_options;
6994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006996 video_source_.IncomingCapturedFrame(
6997 CreateFrame(1, kFrameWidth, kFrameHeight));
6998 WaitForEncodedFrame(1);
6999 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7000 .low_encode_usage_threshold_percent,
7001 default_options.low_encode_usage_threshold_percent);
7002 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7003 .high_encode_usage_threshold_percent,
7004 default_options.high_encode_usage_threshold_percent);
7005
7006 CpuOveruseOptions hardware_options;
7007 hardware_options.low_encode_usage_threshold_percent = 150;
7008 hardware_options.high_encode_usage_threshold_percent = 200;
7009 fake_encoder_.SetIsHardwareAccelerated(true);
7010
7011 video_source_.IncomingCapturedFrame(
7012 CreateFrame(2, kFrameWidth, kFrameHeight));
7013 WaitForEncodedFrame(2);
7014
7015 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7016 .low_encode_usage_threshold_percent,
7017 hardware_options.low_encode_usage_threshold_percent);
7018 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7019 .high_encode_usage_threshold_percent,
7020 hardware_options.high_encode_usage_threshold_percent);
7021
7022 video_stream_encoder_->Stop();
7023}
7024
Niels Möller6bb5ab92019-01-11 11:11:10 +01007025TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7026 const int kFrameWidth = 320;
7027 const int kFrameHeight = 240;
7028 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007029 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007030 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7031
Henrik Boström381d1092020-05-12 18:49:07 +02007032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007034
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007035 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007036 max_framerate_ = kFps;
7037
7038 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7039 fake_encoder_.SimulateOvershoot(1.0);
7040 int num_dropped = 0;
7041 for (int i = 0; i < kNumFramesInRun; ++i) {
7042 video_source_.IncomingCapturedFrame(
7043 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7044 // Wait up to two frame durations for a frame to arrive.
7045 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7046 ++num_dropped;
7047 }
7048 timestamp_ms += 1000 / kFps;
7049 }
7050
Erik Språnga8d48ab2019-02-08 14:17:40 +01007051 // Framerate should be measured to be near the expected target rate.
7052 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7053
7054 // Frame drops should be within 5% of expected 0%.
7055 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007056
7057 // Make encoder produce frames at double the expected bitrate during 3 seconds
7058 // of video, verify number of drops. Rate needs to be slightly changed in
7059 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007060 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007061 const RateControlSettings trials =
7062 RateControlSettings::ParseFromFieldTrials();
7063 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007064 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007065 // frame dropping since the adjuter will try to just lower the target
7066 // bitrate rather than drop frames. If network headroom can be used, it
7067 // doesn't push back as hard so we don't need quite as much overshoot.
7068 // These numbers are unfortunately a bit magical but there's not trivial
7069 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007070 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007071 }
7072 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007074 kTargetBitrate + DataRate::KilobitsPerSec(1),
7075 kTargetBitrate + DataRate::KilobitsPerSec(1),
7076 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007077 num_dropped = 0;
7078 for (int i = 0; i < kNumFramesInRun; ++i) {
7079 video_source_.IncomingCapturedFrame(
7080 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7081 // Wait up to two frame durations for a frame to arrive.
7082 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7083 ++num_dropped;
7084 }
7085 timestamp_ms += 1000 / kFps;
7086 }
7087
Henrik Boström381d1092020-05-12 18:49:07 +02007088 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007089 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007090
7091 // Target framerate should be still be near the expected target, despite
7092 // the frame drops.
7093 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7094
7095 // Frame drops should be within 5% of expected 50%.
7096 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007097
7098 video_stream_encoder_->Stop();
7099}
7100
7101TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7102 const int kFrameWidth = 320;
7103 const int kFrameHeight = 240;
7104 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007105 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007106
7107 ASSERT_GT(max_framerate_, kActualInputFps);
7108
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007109 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007110 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007112 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007113
7114 // Insert 3 seconds of video, with an input fps lower than configured max.
7115 for (int i = 0; i < kActualInputFps * 3; ++i) {
7116 video_source_.IncomingCapturedFrame(
7117 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7118 // Wait up to two frame durations for a frame to arrive.
7119 WaitForEncodedFrame(timestamp_ms);
7120 timestamp_ms += 1000 / kActualInputFps;
7121 }
7122
7123 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7124
7125 video_stream_encoder_->Stop();
7126}
7127
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007128TEST_F(VideoStreamEncoderBlockedTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007129 VideoFrame::UpdateRect rect;
Henrik Boström381d1092020-05-12 18:49:07 +02007130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007131 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007132
7133 fake_encoder_.BlockNextEncode();
7134 video_source_.IncomingCapturedFrame(
7135 CreateFrameWithUpdatedPixel(1, nullptr, 0));
7136 WaitForEncodedFrame(1);
7137 // On the very first frame full update should be forced.
7138 rect = fake_encoder_.GetLastUpdateRect();
7139 EXPECT_EQ(rect.offset_x, 0);
7140 EXPECT_EQ(rect.offset_y, 0);
7141 EXPECT_EQ(rect.height, codec_height_);
7142 EXPECT_EQ(rect.width, codec_width_);
7143 // Here, the encoder thread will be blocked in the TestEncoder waiting for a
7144 // call to ContinueEncode.
7145 video_source_.IncomingCapturedFrame(
7146 CreateFrameWithUpdatedPixel(2, nullptr, 1));
7147 ExpectDroppedFrame();
7148 video_source_.IncomingCapturedFrame(
7149 CreateFrameWithUpdatedPixel(3, nullptr, 10));
7150 ExpectDroppedFrame();
7151 fake_encoder_.ContinueEncode();
7152 WaitForEncodedFrame(3);
7153 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7154 rect = fake_encoder_.GetLastUpdateRect();
7155 EXPECT_EQ(rect.offset_x, 1);
7156 EXPECT_EQ(rect.offset_y, 0);
7157 EXPECT_EQ(rect.width, 10);
7158 EXPECT_EQ(rect.height, 1);
7159
7160 video_source_.IncomingCapturedFrame(
7161 CreateFrameWithUpdatedPixel(4, nullptr, 0));
7162 WaitForEncodedFrame(4);
7163 // Previous frame was encoded, so no accumulation should happen.
7164 rect = fake_encoder_.GetLastUpdateRect();
7165 EXPECT_EQ(rect.offset_x, 0);
7166 EXPECT_EQ(rect.offset_y, 0);
7167 EXPECT_EQ(rect.width, 1);
7168 EXPECT_EQ(rect.height, 1);
7169
7170 video_stream_encoder_->Stop();
7171}
7172
Erik Språngd7329ca2019-02-21 21:19:53 +01007173TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007175 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007176
7177 // First frame is always keyframe.
7178 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7179 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007180 EXPECT_THAT(
7181 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007182 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007183
7184 // Insert delta frame.
7185 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7186 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007187 EXPECT_THAT(
7188 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007189 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007190
7191 // Request next frame be a key-frame.
7192 video_stream_encoder_->SendKeyFrame();
7193 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7194 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007195 EXPECT_THAT(
7196 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007197 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007198
7199 video_stream_encoder_->Stop();
7200}
7201
7202TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7203 // Setup simulcast with three streams.
7204 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007206 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7207 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007208 // Wait for all three layers before triggering event.
7209 sink_.SetNumExpectedLayers(3);
7210
7211 // First frame is always keyframe.
7212 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7213 WaitForEncodedFrame(1);
7214 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007215 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7216 VideoFrameType::kVideoFrameKey,
7217 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007218
7219 // Insert delta frame.
7220 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7221 WaitForEncodedFrame(2);
7222 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007223 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7224 VideoFrameType::kVideoFrameDelta,
7225 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007226
7227 // Request next frame be a key-frame.
7228 // Only first stream is configured to produce key-frame.
7229 video_stream_encoder_->SendKeyFrame();
7230 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7231 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007232
7233 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7234 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007235 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007236 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007237 VideoFrameType::kVideoFrameKey,
7238 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007239
7240 video_stream_encoder_->Stop();
7241}
7242
7243TEST_F(VideoStreamEncoderTest, RequestKeyframeInternalSource) {
7244 // Configure internal source factory and setup test again.
7245 encoder_factory_.SetHasInternalSource(true);
7246 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007247 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007248 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007249
7250 // Call encoder directly, simulating internal source where encoded frame
7251 // callback in VideoStreamEncoder is called despite no OnFrame().
7252 fake_encoder_.InjectFrame(CreateFrame(1, nullptr), true);
7253 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007254 EXPECT_THAT(
7255 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007256 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007257
Niels Möller8f7ce222019-03-21 15:43:58 +01007258 const std::vector<VideoFrameType> kDeltaFrame = {
7259 VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01007260 // Need to set timestamp manually since manually for injected frame.
7261 VideoFrame frame = CreateFrame(101, nullptr);
7262 frame.set_timestamp(101);
7263 fake_encoder_.InjectFrame(frame, false);
7264 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007265 EXPECT_THAT(
7266 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007267 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007268
7269 // Request key-frame. The forces a dummy frame down into the encoder.
7270 fake_encoder_.ExpectNullFrame();
7271 video_stream_encoder_->SendKeyFrame();
7272 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Niels Möller8f7ce222019-03-21 15:43:58 +01007273 EXPECT_THAT(
7274 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007275 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007276
7277 video_stream_encoder_->Stop();
7278}
Erik Språngb7cb7b52019-02-26 15:52:33 +01007279
7280TEST_F(VideoStreamEncoderTest, AdjustsTimestampInternalSource) {
7281 // Configure internal source factory and setup test again.
7282 encoder_factory_.SetHasInternalSource(true);
7283 ResetEncoder("VP8", 1, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007286
7287 int64_t timestamp = 1;
7288 EncodedImage image;
Erik Språngb7cb7b52019-02-26 15:52:33 +01007289 image.capture_time_ms_ = ++timestamp;
7290 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
7291 const int64_t kEncodeFinishDelayMs = 10;
7292 image.timing_.encode_start_ms = timestamp;
7293 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007294 fake_encoder_.InjectEncodedImage(image, /*codec_specific_info=*/nullptr);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007295 // Wait for frame without incrementing clock.
7296 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7297 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
7298 // capture timestamp should be kEncodeFinishDelayMs in the past.
7299 EXPECT_EQ(sink_.GetLastCaptureTimeMs(),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007300 CurrentTimeMs() - kEncodeFinishDelayMs);
Erik Språngb7cb7b52019-02-26 15:52:33 +01007301
7302 video_stream_encoder_->Stop();
7303}
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007304
7305TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007306 // SPS contains VUI with restrictions on the maximum number of reordered
7307 // pictures, there is no need to rewrite the bitstream to enable faster
7308 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007309 ResetEncoder("H264", 1, 1, 1, false);
7310
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007311 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007312 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007314
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007315 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007316 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007317
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007318 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7319 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007320
7321 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007322 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007323
7324 video_stream_encoder_->Stop();
7325}
7326
7327TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007328 // SPS does not contain VUI, the bitstream is will be rewritten with added
7329 // VUI with restrictions on the maximum number of reordered pictures to
7330 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007331 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7332 0x00, 0x00, 0x03, 0x03, 0xF4,
7333 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007334 ResetEncoder("H264", 1, 1, 1, false);
7335
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007338 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007339
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007340 fake_encoder_.SetEncodedImageData(
7341 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007342
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007343 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7344 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345
7346 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007347 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007348
7349 video_stream_encoder_->Stop();
7350}
7351
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007352TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7353 const int kFrameWidth = 1280;
7354 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007355 const DataRate kTargetBitrate =
7356 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007357
Henrik Boström381d1092020-05-12 18:49:07 +02007358 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007359 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007360 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7361
7362 // Insert a first video frame. It should be dropped because of downscale in
7363 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007364 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007365 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7366 frame.set_rotation(kVideoRotation_270);
7367 video_source_.IncomingCapturedFrame(frame);
7368
7369 ExpectDroppedFrame();
7370
7371 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007372 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007373 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7374 frame.set_rotation(kVideoRotation_90);
7375 video_source_.IncomingCapturedFrame(frame);
7376
7377 WaitForEncodedFrame(timestamp_ms);
7378 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7379
7380 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007381 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007382 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7383 frame.set_rotation(kVideoRotation_180);
7384 video_source_.IncomingCapturedFrame(frame);
7385
7386 WaitForEncodedFrame(timestamp_ms);
7387 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7388
7389 video_stream_encoder_->Stop();
7390}
7391
Erik Språng5056af02019-09-02 15:53:11 +02007392TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7393 const int kFrameWidth = 320;
7394 const int kFrameHeight = 180;
7395
7396 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007398 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7399 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7400 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007401 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007402 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007403 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007404
7405 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007406 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007407 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7408 frame.set_rotation(kVideoRotation_270);
7409 video_source_.IncomingCapturedFrame(frame);
7410 WaitForEncodedFrame(timestamp_ms);
7411
7412 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007413 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007414 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7415 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007417 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007418 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007419 /*link_allocation=*/target_rate,
7420 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007421 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007422 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007423 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7424
7425 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7426 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7427 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007428 DataRate allocation_sum =
7429 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007430 EXPECT_EQ(min_rate, allocation_sum);
7431 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7432
7433 video_stream_encoder_->Stop();
7434}
7435
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007436TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007438 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007439 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007440 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007441 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7442 WaitForEncodedFrame(1);
7443
7444 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7445 ASSERT_TRUE(prev_rate_settings.has_value());
7446 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7447 kDefaultFramerate);
7448
7449 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7450 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7451 timestamp_ms += 1000 / kDefaultFramerate;
7452 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7453 WaitForEncodedFrame(timestamp_ms);
7454 }
7455 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7456 kDefaultFramerate);
7457 // Capture larger frame to trigger a reconfigure.
7458 codec_height_ *= 2;
7459 codec_width_ *= 2;
7460 timestamp_ms += 1000 / kDefaultFramerate;
7461 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7462 WaitForEncodedFrame(timestamp_ms);
7463
7464 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7465 auto current_rate_settings =
7466 fake_encoder_.GetAndResetLastRateControlSettings();
7467 // Ensure we have actually reconfigured twice
7468 // The rate settings should have been set again even though
7469 // they haven't changed.
7470 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007471 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007472
7473 video_stream_encoder_->Stop();
7474}
7475
philipeld9cc8c02019-09-16 14:53:40 +02007476struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007477 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7478 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7479 MOCK_METHOD(void,
7480 RequestEncoderSwitch,
7481 (const webrtc::SdpVideoFormat& format),
7482 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007483};
7484
philipel9b058032020-02-10 11:30:00 +01007485TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7486 constexpr int kDontCare = 100;
7487 StrictMock<MockEncoderSelector> encoder_selector;
7488 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7489 &fake_encoder_, &encoder_selector);
7490 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7491
7492 // Reset encoder for new configuration to take effect.
7493 ConfigureEncoder(video_encoder_config_.Copy());
7494
7495 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7496
7497 video_source_.IncomingCapturedFrame(
7498 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007499 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007500 video_stream_encoder_->Stop();
7501
7502 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7503 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007504 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7505 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007506 video_stream_encoder_.reset();
7507}
7508
7509TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7510 constexpr int kDontCare = 100;
7511
7512 NiceMock<MockEncoderSelector> encoder_selector;
7513 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7514 video_send_config_.encoder_settings.encoder_switch_request_callback =
7515 &switch_callback;
7516 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7517 &fake_encoder_, &encoder_selector);
7518 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7519
7520 // Reset encoder for new configuration to take effect.
7521 ConfigureEncoder(video_encoder_config_.Copy());
7522
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007523 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007524 .WillByDefault(Return(SdpVideoFormat("AV1")));
7525 EXPECT_CALL(switch_callback,
7526 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7527 Field(&SdpVideoFormat::name, "AV1"))));
7528
Henrik Boström381d1092020-05-12 18:49:07 +02007529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007530 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7531 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7532 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007533 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007534 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007535 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007536 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007537
7538 video_stream_encoder_->Stop();
7539}
7540
7541TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7542 constexpr int kSufficientBitrateToNotDrop = 1000;
7543 constexpr int kDontCare = 100;
7544
7545 NiceMock<MockVideoEncoder> video_encoder;
7546 NiceMock<MockEncoderSelector> encoder_selector;
7547 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7548 video_send_config_.encoder_settings.encoder_switch_request_callback =
7549 &switch_callback;
7550 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7551 &video_encoder, &encoder_selector);
7552 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7553
7554 // Reset encoder for new configuration to take effect.
7555 ConfigureEncoder(video_encoder_config_.Copy());
7556
7557 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7558 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7559 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007561 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7562 /*stable_target_bitrate=*/
7563 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7564 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007565 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007566 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007567 /*cwnd_reduce_ratio=*/0);
7568
7569 ON_CALL(video_encoder, Encode(_, _))
7570 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7571 ON_CALL(encoder_selector, OnEncoderBroken())
7572 .WillByDefault(Return(SdpVideoFormat("AV2")));
7573
7574 rtc::Event encode_attempted;
7575 EXPECT_CALL(switch_callback,
7576 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7577 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7578 EXPECT_EQ(format.name, "AV2");
7579 encode_attempted.Set();
7580 });
7581
7582 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7583 encode_attempted.Wait(3000);
7584
Markus Handell28c71802021-11-08 10:11:55 +01007585 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007586
philipel9b058032020-02-10 11:30:00 +01007587 video_stream_encoder_->Stop();
7588
7589 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7590 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007591 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7592 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007593 video_stream_encoder_.reset();
7594}
7595
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007596TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007597 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007598 const int kFrameWidth = 320;
7599 const int kFrameHeight = 180;
7600
7601 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007602 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007604 /*target_bitrate=*/rate,
7605 /*stable_target_bitrate=*/rate,
7606 /*link_allocation=*/rate,
7607 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007608 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007609 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007610
7611 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007612 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007613 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7614 frame.set_rotation(kVideoRotation_270);
7615 video_source_.IncomingCapturedFrame(frame);
7616 WaitForEncodedFrame(timestamp_ms);
7617 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7618
7619 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007620 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007622 /*target_bitrate=*/new_stable_rate,
7623 /*stable_target_bitrate=*/new_stable_rate,
7624 /*link_allocation=*/rate,
7625 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007626 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007627 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007628 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7629 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7630 video_stream_encoder_->Stop();
7631}
7632
7633TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007634 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007635 const int kFrameWidth = 320;
7636 const int kFrameHeight = 180;
7637
7638 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007639 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007640 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007641 /*target_bitrate=*/rate,
7642 /*stable_target_bitrate=*/rate,
7643 /*link_allocation=*/rate,
7644 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007645 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007646 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007647
7648 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007649 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007650 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7651 frame.set_rotation(kVideoRotation_270);
7652 video_source_.IncomingCapturedFrame(frame);
7653 WaitForEncodedFrame(timestamp_ms);
7654 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7655
7656 // Set a higher target rate without changing the link_allocation. Should not
7657 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007658 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007660 /*target_bitrate=*/rate,
7661 /*stable_target_bitrate=*/new_stable_rate,
7662 /*link_allocation=*/rate,
7663 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007664 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007665 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7667 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7668 video_stream_encoder_->Stop();
7669}
7670
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007671TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7672 test::ScopedFieldTrials field_trials(
7673 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7674 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7675 const int kFramerateFps = 30;
7676 const int kWidth = 1920;
7677 const int kHeight = 1080;
7678 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7679 // Works on screenshare mode.
7680 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7681 // We rely on the automatic resolution adaptation, but we handle framerate
7682 // adaptation manually by mocking the stats proxy.
7683 video_source_.set_adaptation_enabled(true);
7684
7685 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007687 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007688 video_stream_encoder_->SetSource(&video_source_,
7689 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007690 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007691
7692 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7693 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7694
7695 // Pass enough frames with the full update to trigger animation detection.
7696 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007697 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007698 frame.set_ntp_time_ms(timestamp_ms);
7699 frame.set_timestamp_us(timestamp_ms * 1000);
7700 video_source_.IncomingCapturedFrame(frame);
7701 WaitForEncodedFrame(timestamp_ms);
7702 }
7703
7704 // Resolution should be limited.
7705 rtc::VideoSinkWants expected;
7706 expected.max_framerate_fps = kFramerateFps;
7707 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007708 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007709
7710 // Pass one frame with no known update.
7711 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007712 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007713 frame.set_ntp_time_ms(timestamp_ms);
7714 frame.set_timestamp_us(timestamp_ms * 1000);
7715 frame.clear_update_rect();
7716
7717 video_source_.IncomingCapturedFrame(frame);
7718 WaitForEncodedFrame(timestamp_ms);
7719
7720 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007721 EXPECT_THAT(video_source_.sink_wants(),
7722 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007723
7724 video_stream_encoder_->Stop();
7725}
7726
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007727TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7728 const int kWidth = 720; // 540p adapted down.
7729 const int kHeight = 405;
7730 const int kNumFrames = 3;
7731 // Works on screenshare mode.
7732 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7733 /*num_spatial_layers=*/2, /*screenshare=*/true);
7734
7735 video_source_.set_adaptation_enabled(true);
7736
7737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007738 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007739
7740 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7741
7742 // Pass enough frames with the full update to trigger animation detection.
7743 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007744 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007745 frame.set_ntp_time_ms(timestamp_ms);
7746 frame.set_timestamp_us(timestamp_ms * 1000);
7747 video_source_.IncomingCapturedFrame(frame);
7748 WaitForEncodedFrame(timestamp_ms);
7749 }
7750
7751 video_stream_encoder_->Stop();
7752}
7753
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007754TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7755 const float downscale_factors[] = {4.0, 2.0, 1.0};
7756 const int number_layers =
7757 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7758 VideoEncoderConfig config;
7759 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7760 for (int i = 0; i < number_layers; ++i) {
7761 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7762 config.simulcast_layers[i].active = true;
7763 }
7764 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007765 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007766 "VP8", /*max qp*/ 56, /*screencast*/ false,
7767 /*screenshare enabled*/ false);
7768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007769 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7770 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007771
7772 // First initialization.
7773 // Encoder should be initialized. Next frame should be key frame.
7774 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7775 sink_.SetNumExpectedLayers(number_layers);
7776 int64_t timestamp_ms = kFrameIntervalMs;
7777 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7778 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007779 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007780 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7781 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7782 VideoFrameType::kVideoFrameKey,
7783 VideoFrameType::kVideoFrameKey}));
7784
7785 // Disable top layer.
7786 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7787 config.simulcast_layers[number_layers - 1].active = false;
7788 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7789 sink_.SetNumExpectedLayers(number_layers - 1);
7790 timestamp_ms += kFrameIntervalMs;
7791 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7792 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007793 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007794 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7795 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7796 VideoFrameType::kVideoFrameDelta,
7797 VideoFrameType::kVideoFrameDelta}));
7798
7799 // Re-enable top layer.
7800 // Encoder should be re-initialized. Next frame should be key frame.
7801 config.simulcast_layers[number_layers - 1].active = true;
7802 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7803 sink_.SetNumExpectedLayers(number_layers);
7804 timestamp_ms += kFrameIntervalMs;
7805 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7806 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007807 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007808 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7809 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7810 VideoFrameType::kVideoFrameKey,
7811 VideoFrameType::kVideoFrameKey}));
7812
7813 // Top layer max rate change.
7814 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7815 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7816 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7817 sink_.SetNumExpectedLayers(number_layers);
7818 timestamp_ms += kFrameIntervalMs;
7819 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7820 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007821 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007822 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7823 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7824 VideoFrameType::kVideoFrameDelta,
7825 VideoFrameType::kVideoFrameDelta}));
7826
7827 // Top layer resolution change.
7828 // Encoder should be re-initialized. Next frame should be key frame.
7829 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7830 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7831 sink_.SetNumExpectedLayers(number_layers);
7832 timestamp_ms += kFrameIntervalMs;
7833 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7834 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007835 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007836 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7837 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7838 VideoFrameType::kVideoFrameKey,
7839 VideoFrameType::kVideoFrameKey}));
7840 video_stream_encoder_->Stop();
7841}
7842
Henrik Boström1124ed12021-02-25 10:30:39 +01007843TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7844 const int kFrameWidth = 1280;
7845 const int kFrameHeight = 720;
7846
7847 SetUp();
7848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007850
7851 // Capturing a frame should reconfigure the encoder and expose the encoder
7852 // resolution, which is the same as the input frame.
7853 int64_t timestamp_ms = kFrameIntervalMs;
7854 video_source_.IncomingCapturedFrame(
7855 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7856 WaitForEncodedFrame(timestamp_ms);
7857 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7858 EXPECT_THAT(video_source_.sink_wants().resolutions,
7859 ::testing::ElementsAreArray(
7860 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7861
7862 video_stream_encoder_->Stop();
7863}
7864
7865TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7866 // Pick downscale factors such that we never encode at full resolution - this
7867 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007868 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007869 // encoder should not ask for the frame resolution. This allows video frames
7870 // to have the appearence of one resolution but optimize its internal buffers
7871 // for what is actually encoded.
7872 const size_t kNumSimulcastLayers = 3u;
7873 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7874 const int kFrameWidth = 1280;
7875 const int kFrameHeight = 720;
7876 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7877 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7878 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7879 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7880 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7881 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7882
7883 VideoEncoderConfig config;
7884 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7885 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7886 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7887 config.simulcast_layers[i].active = true;
7888 }
7889 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007890 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007891 "VP8", /*max qp*/ 56, /*screencast*/ false,
7892 /*screenshare enabled*/ false);
7893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007894 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7895 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007896
7897 // Capture a frame with all layers active.
7898 int64_t timestamp_ms = kFrameIntervalMs;
7899 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7900 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7901 video_source_.IncomingCapturedFrame(
7902 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7903 WaitForEncodedFrame(timestamp_ms);
7904 // Expect encoded resolutions to match the expected simulcast layers.
7905 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7906 EXPECT_THAT(
7907 video_source_.sink_wants().resolutions,
7908 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7909
7910 // Capture a frame with one of the layers inactive.
7911 timestamp_ms += kFrameIntervalMs;
7912 config.simulcast_layers[2].active = false;
7913 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7914 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7915 video_source_.IncomingCapturedFrame(
7916 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7917 WaitForEncodedFrame(timestamp_ms);
7918
7919 // Expect encoded resolutions to match the expected simulcast layers.
7920 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7921 EXPECT_THAT(video_source_.sink_wants().resolutions,
7922 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7923
7924 // Capture a frame with all but one layer turned off.
7925 timestamp_ms += kFrameIntervalMs;
7926 config.simulcast_layers[1].active = false;
7927 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7928 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7929 video_source_.IncomingCapturedFrame(
7930 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7931 WaitForEncodedFrame(timestamp_ms);
7932
7933 // Expect encoded resolutions to match the expected simulcast layers.
7934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7935 EXPECT_THAT(video_source_.sink_wants().resolutions,
7936 ::testing::ElementsAreArray({kLayer0Size}));
7937
7938 video_stream_encoder_->Stop();
7939}
7940
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007941TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007942 ResetEncoder("VP8", 1, 1, 1, false);
7943
Niels Möller8b692902021-06-14 12:04:57 +02007944 // Force encoder reconfig.
7945 video_source_.IncomingCapturedFrame(
7946 CreateFrame(1, codec_width_, codec_height_));
7947 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7948
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007949 // Set QP on encoded frame and pass the frame to encode complete callback.
7950 // Since QP is present QP parsing won't be triggered and the original value
7951 // should be kept.
7952 EncodedImage encoded_image;
7953 encoded_image.qp_ = 123;
7954 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7955 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7956 CodecSpecificInfo codec_info;
7957 codec_info.codecType = kVideoCodecVP8;
7958 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7959 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7960 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7961 video_stream_encoder_->Stop();
7962}
7963
7964TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007965 ResetEncoder("VP8", 1, 1, 1, false);
7966
Niels Möller8b692902021-06-14 12:04:57 +02007967 // Force encoder reconfig.
7968 video_source_.IncomingCapturedFrame(
7969 CreateFrame(1, codec_width_, codec_height_));
7970 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7971
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007972 // Pass an encoded frame without QP to encode complete callback. QP should be
7973 // parsed and set.
7974 EncodedImage encoded_image;
7975 encoded_image.qp_ = -1;
7976 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7977 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7978 CodecSpecificInfo codec_info;
7979 codec_info.codecType = kVideoCodecVP8;
7980 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7981 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7982 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7983 video_stream_encoder_->Stop();
7984}
7985
7986TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7987 webrtc::test::ScopedFieldTrials field_trials(
7988 "WebRTC-QpParsingKillSwitch/Enabled/");
7989
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007990 ResetEncoder("VP8", 1, 1, 1, false);
7991
Niels Möller8b692902021-06-14 12:04:57 +02007992 // Force encoder reconfig.
7993 video_source_.IncomingCapturedFrame(
7994 CreateFrame(1, codec_width_, codec_height_));
7995 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7996
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007997 EncodedImage encoded_image;
7998 encoded_image.qp_ = -1;
7999 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8000 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8001 CodecSpecificInfo codec_info;
8002 codec_info.codecType = kVideoCodecVP8;
8003 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8004 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8005 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8006 video_stream_encoder_->Stop();
8007}
8008
Sergey Silkind19e3b92021-03-16 10:05:30 +00008009TEST_F(VideoStreamEncoderTest,
8010 QualityScalingNotAllowed_QualityScalingDisabled) {
8011 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8012
8013 // Disable scaling settings in encoder info.
8014 fake_encoder_.SetQualityScaling(false);
8015 // Disable quality scaling in encoder config.
8016 video_encoder_config.is_quality_scaling_allowed = false;
8017 ConfigureEncoder(std::move(video_encoder_config));
8018
8019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008020 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008021
8022 test::FrameForwarder source;
8023 video_stream_encoder_->SetSource(
8024 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8025 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8026 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8027
8028 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8029 WaitForEncodedFrame(1);
8030 video_stream_encoder_->TriggerQualityLow();
8031 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8032
8033 video_stream_encoder_->Stop();
8034}
8035
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008036TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8037 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8038
8039 // Disable scaling settings in encoder info.
8040 fake_encoder_.SetQualityScaling(false);
8041 // Set QP trusted in encoder info.
8042 fake_encoder_.SetIsQpTrusted(true);
8043 // Enable quality scaling in encoder config.
8044 video_encoder_config.is_quality_scaling_allowed = false;
8045 ConfigureEncoder(std::move(video_encoder_config));
8046
8047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008048 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008049
8050 test::FrameForwarder source;
8051 video_stream_encoder_->SetSource(
8052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8053 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8054 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8055
8056 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8057 WaitForEncodedFrame(1);
8058 video_stream_encoder_->TriggerQualityLow();
8059 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8060
8061 video_stream_encoder_->Stop();
8062}
8063
Shuhai Pengf2707702021-09-29 17:19:44 +08008064TEST_F(VideoStreamEncoderTest,
8065 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8066 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8067
8068 // Disable scaling settings in encoder info.
8069 fake_encoder_.SetQualityScaling(false);
8070 // Set QP trusted in encoder info.
8071 fake_encoder_.SetIsQpTrusted(true);
8072 // Enable quality scaling in encoder config.
8073 video_encoder_config.is_quality_scaling_allowed = false;
8074 ConfigureEncoder(std::move(video_encoder_config));
8075
8076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008077 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008078
8079 test::FrameForwarder source;
8080 video_stream_encoder_->SetSource(
8081 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8082 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8083 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8084
8085 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8086 WaitForEncodedFrame(1);
8087 video_stream_encoder_->TriggerQualityLow();
8088 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8089
8090 video_stream_encoder_->Stop();
8091}
8092
8093TEST_F(VideoStreamEncoderTest,
8094 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8095 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8096
8097 // Disable scaling settings in encoder info.
8098 fake_encoder_.SetQualityScaling(false);
8099 // Set QP trusted in encoder info.
8100 fake_encoder_.SetIsQpTrusted(false);
8101 // Enable quality scaling in encoder config.
8102 video_encoder_config.is_quality_scaling_allowed = false;
8103 ConfigureEncoder(std::move(video_encoder_config));
8104
8105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008106 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008107
8108 test::FrameForwarder source;
8109 video_stream_encoder_->SetSource(
8110 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8111 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8112 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8113
8114 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8115 WaitForEncodedFrame(1);
8116 video_stream_encoder_->TriggerQualityLow();
8117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8118
8119 video_stream_encoder_->Stop();
8120}
8121
8122TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8123 // Set QP trusted in encoder info.
8124 fake_encoder_.SetIsQpTrusted(false);
8125
8126 const int MinEncBitrateKbps = 30;
8127 const int MaxEncBitrateKbps = 100;
8128 const int MinStartBitrateKbp = 50;
8129 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8130 /*frame_size_pixels=*/codec_width_ * codec_height_,
8131 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8132 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8133 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8134
8135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008136 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008137
8138 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8139
8140 VideoEncoderConfig video_encoder_config;
8141 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8142 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8143 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8144 MinEncBitrateKbps * 1000;
8145 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8146 kMaxPayloadLength);
8147
8148 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8149 WaitForEncodedFrame(1);
8150 EXPECT_EQ(
8151 MaxEncBitrateKbps,
8152 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8153 EXPECT_EQ(
8154 MinEncBitrateKbps,
8155 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8156
8157 video_stream_encoder_->Stop();
8158}
8159
8160TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8161 // Set QP trusted in encoder info.
8162 fake_encoder_.SetIsQpTrusted(false);
8163
8164 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8165 EncoderInfoSettings::
8166 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8167 codec_width_ * codec_height_,
8168 EncoderInfoSettings::
8169 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8170 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8171
8172 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8173 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8174 const int TargetEncBitrate = MaxEncBitrate;
8175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8176 DataRate::BitsPerSec(TargetEncBitrate),
8177 DataRate::BitsPerSec(TargetEncBitrate),
8178 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8179
8180 VideoEncoderConfig video_encoder_config;
8181 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8182 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8183 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8184 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8185 kMaxPayloadLength);
8186
8187 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8188 WaitForEncodedFrame(1);
8189 EXPECT_EQ(
8190 MaxEncBitrate / 1000,
8191 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8192 EXPECT_EQ(
8193 MinEncBitrate / 1000,
8194 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8195
8196 video_stream_encoder_->Stop();
8197}
8198
Sergey Silkind19e3b92021-03-16 10:05:30 +00008199#if !defined(WEBRTC_IOS)
8200// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8201// disabled by default on iOS.
8202TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8203 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8204
8205 // Disable scaling settings in encoder info.
8206 fake_encoder_.SetQualityScaling(false);
8207 // Enable quality scaling in encoder config.
8208 video_encoder_config.is_quality_scaling_allowed = true;
8209 ConfigureEncoder(std::move(video_encoder_config));
8210
8211 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008212 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008213
8214 test::FrameForwarder source;
8215 video_stream_encoder_->SetSource(
8216 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8217 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8218 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8219
8220 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8221 WaitForEncodedFrame(1);
8222 video_stream_encoder_->TriggerQualityLow();
8223 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8224
8225 video_stream_encoder_->Stop();
8226}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008227
8228TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8229 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8230
8231 // Disable scaling settings in encoder info.
8232 fake_encoder_.SetQualityScaling(false);
8233 // Set QP trusted in encoder info.
8234 fake_encoder_.SetIsQpTrusted(true);
8235 // Enable quality scaling in encoder config.
8236 video_encoder_config.is_quality_scaling_allowed = true;
8237 ConfigureEncoder(std::move(video_encoder_config));
8238
8239 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008240 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008241
8242 test::FrameForwarder source;
8243 video_stream_encoder_->SetSource(
8244 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8245 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8247
8248 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8249 WaitForEncodedFrame(1);
8250 video_stream_encoder_->TriggerQualityLow();
8251 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8252
8253 video_stream_encoder_->Stop();
8254}
Shuhai Pengf2707702021-09-29 17:19:44 +08008255
8256TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8257 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8258
8259 // Disable scaling settings in encoder info.
8260 fake_encoder_.SetQualityScaling(false);
8261 // Set QP not trusted in encoder info.
8262 fake_encoder_.SetIsQpTrusted(false);
8263 // Enable quality scaling in encoder config.
8264 video_encoder_config.is_quality_scaling_allowed = true;
8265 ConfigureEncoder(std::move(video_encoder_config));
8266
8267 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008268 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008269
8270 test::FrameForwarder source;
8271 video_stream_encoder_->SetSource(
8272 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8273 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8274 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8275
8276 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8277 WaitForEncodedFrame(1);
8278 video_stream_encoder_->TriggerQualityLow();
8279 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8280 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8281 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8282
8283 video_stream_encoder_->Stop();
8284}
8285
8286TEST_F(VideoStreamEncoderTest,
8287 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8288 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8289
8290 // Disable scaling settings in encoder info.
8291 fake_encoder_.SetQualityScaling(false);
8292 // Set QP trusted in encoder info.
8293 fake_encoder_.SetIsQpTrusted(true);
8294 // Enable quality scaling in encoder config.
8295 video_encoder_config.is_quality_scaling_allowed = true;
8296 ConfigureEncoder(std::move(video_encoder_config));
8297
8298 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008299 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008300
8301 test::FrameForwarder source;
8302 video_stream_encoder_->SetSource(
8303 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8304 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8305 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8306
8307 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8308 WaitForEncodedFrame(1);
8309 video_stream_encoder_->TriggerQualityLow();
8310 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8311 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8312
8313 video_stream_encoder_->Stop();
8314}
8315
8316TEST_F(VideoStreamEncoderTest,
8317 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8318 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8319
8320 // Disable scaling settings in encoder info.
8321 fake_encoder_.SetQualityScaling(false);
8322 // Set QP trusted in encoder info.
8323 fake_encoder_.SetIsQpTrusted(false);
8324 // Enable quality scaling in encoder config.
8325 video_encoder_config.is_quality_scaling_allowed = true;
8326 ConfigureEncoder(std::move(video_encoder_config));
8327
8328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008329 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008330
8331 test::FrameForwarder source;
8332 video_stream_encoder_->SetSource(
8333 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8334 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8335 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8336
8337 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8338 WaitForEncodedFrame(1);
8339 video_stream_encoder_->TriggerQualityLow();
8340 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8341
8342 video_stream_encoder_->Stop();
8343}
8344
Sergey Silkind19e3b92021-03-16 10:05:30 +00008345#endif
8346
Henrik Boström56db9ff2021-03-24 09:06:45 +01008347// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8348class VideoStreamEncoderWithRealEncoderTest
8349 : public VideoStreamEncoderTest,
8350 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8351 public:
8352 VideoStreamEncoderWithRealEncoderTest()
8353 : VideoStreamEncoderTest(),
8354 codec_type_(std::get<0>(GetParam())),
8355 allow_i420_conversion_(std::get<1>(GetParam())) {}
8356
8357 void SetUp() override {
8358 VideoStreamEncoderTest::SetUp();
8359 std::unique_ptr<VideoEncoder> encoder;
8360 switch (codec_type_) {
8361 case kVideoCodecVP8:
8362 encoder = VP8Encoder::Create();
8363 break;
8364 case kVideoCodecVP9:
8365 encoder = VP9Encoder::Create();
8366 break;
8367 case kVideoCodecAV1:
8368 encoder = CreateLibaomAv1Encoder();
8369 break;
8370 case kVideoCodecH264:
8371 encoder =
8372 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8373 break;
8374 case kVideoCodecMultiplex:
8375 mock_encoder_factory_for_multiplex_ =
8376 std::make_unique<MockVideoEncoderFactory>();
8377 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8378 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8379 .WillRepeatedly([] { return VP8Encoder::Create(); });
8380 encoder = std::make_unique<MultiplexEncoderAdapter>(
8381 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8382 false);
8383 break;
8384 default:
8385 RTC_NOTREACHED();
8386 }
8387 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8388 }
8389
8390 void TearDown() override {
8391 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008392 // Ensure `video_stream_encoder_` is destroyed before
8393 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008394 video_stream_encoder_.reset();
8395 VideoStreamEncoderTest::TearDown();
8396 }
8397
8398 protected:
8399 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8400 std::unique_ptr<VideoEncoder> encoder) {
8401 // Configure VSE to use the encoder.
8402 encoder_ = std::move(encoder);
8403 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8404 encoder_.get(), &encoder_selector_);
8405 video_send_config_.encoder_settings.encoder_factory =
8406 encoder_proxy_factory_.get();
8407 VideoEncoderConfig video_encoder_config;
8408 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8409 video_encoder_config_ = video_encoder_config.Copy();
8410 ConfigureEncoder(video_encoder_config_.Copy());
8411
8412 // Set bitrate to ensure frame is not dropped.
8413 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008414 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008415 }
8416
8417 const VideoCodecType codec_type_;
8418 const bool allow_i420_conversion_;
8419 NiceMock<MockEncoderSelector> encoder_selector_;
8420 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8421 std::unique_ptr<VideoEncoder> encoder_;
8422 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8423};
8424
8425TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8426 auto native_i420_frame = test::CreateMappableNativeFrame(
8427 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8428 video_source_.IncomingCapturedFrame(native_i420_frame);
8429 WaitForEncodedFrame(codec_width_, codec_height_);
8430
8431 auto mappable_native_buffer =
8432 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8433 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8434 mappable_native_buffer->GetMappedFramedBuffers();
8435 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8436 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8437 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8438 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8439}
8440
8441TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8442 auto native_nv12_frame = test::CreateMappableNativeFrame(
8443 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8444 video_source_.IncomingCapturedFrame(native_nv12_frame);
8445 WaitForEncodedFrame(codec_width_, codec_height_);
8446
8447 auto mappable_native_buffer =
8448 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8449 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8450 mappable_native_buffer->GetMappedFramedBuffers();
8451 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8452 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8453 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8454 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8455
8456 if (!allow_i420_conversion_) {
8457 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8458 }
8459}
8460
Erik Språng7444b192021-06-02 14:02:13 +02008461TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8462 if (codec_type_ == kVideoCodecMultiplex) {
8463 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8464 return;
8465 }
8466
8467 const size_t kNumSpatialLayers = 3u;
8468 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8469 const int kFrameWidth = 1280;
8470 const int kFrameHeight = 720;
8471 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8472 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8473 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8474 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8475 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8476 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8477
8478 VideoEncoderConfig config;
8479 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8480 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008481 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008482 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8483 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8484 vp9_settings.numberOfTemporalLayers = 3;
8485 vp9_settings.automaticResizeOn = false;
8486 config.encoder_specific_settings =
8487 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8488 vp9_settings);
8489 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8490 /*fps=*/30.0,
8491 /*first_active_layer=*/0,
8492 /*num_spatial_layers=*/3,
8493 /*num_temporal_layers=*/3,
8494 /*is_screenshare=*/false);
8495 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8496 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008497 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008498 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8499 /*fps=*/30.0,
8500 /*first_active_layer=*/0,
8501 /*num_spatial_layers=*/3,
8502 /*num_temporal_layers=*/3,
8503 /*is_screenshare=*/false);
8504 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8505 } else {
8506 // Simulcast for VP8/H264.
8507 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8508 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8509 config.simulcast_layers[i].scale_resolution_down_by =
8510 kDownscaleFactors[i];
8511 config.simulcast_layers[i].active = true;
8512 }
8513 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8514 // Turn off frame dropping to prevent flakiness.
8515 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8516 h264_settings.frameDroppingOn = false;
8517 config.encoder_specific_settings = rtc::make_ref_counted<
8518 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8519 }
8520 }
8521
8522 auto set_layer_active = [&](int layer_idx, bool active) {
8523 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8524 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8525 config.spatial_layers[layer_idx].active = active;
8526 } else {
8527 config.simulcast_layers[layer_idx].active = active;
8528 }
8529 };
8530
8531 config.video_stream_factory =
8532 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8533 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8534 /*screencast*/ false,
8535 /*screenshare enabled*/ false);
8536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008537 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8538 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008539
8540 // Capture a frame with all layers active.
8541 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8542 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8543 int64_t timestamp_ms = kFrameIntervalMs;
8544 video_source_.IncomingCapturedFrame(
8545 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8546
8547 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8548 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8549
8550 // Capture a frame with one of the layers inactive.
8551 set_layer_active(2, false);
8552 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8553 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8554 timestamp_ms += kFrameIntervalMs;
8555 video_source_.IncomingCapturedFrame(
8556 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8557 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8558
8559 // New target bitrates signaled based on lower resolution.
8560 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8562 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8563 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8564
8565 // Re-enable the top layer.
8566 set_layer_active(2, true);
8567 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8568 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8569 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8570
8571 // Bitrate target adjusted back up to enable HD layer...
8572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8573 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8574 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8575 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8576
8577 // ...then add a new frame.
8578 timestamp_ms += kFrameIntervalMs;
8579 video_source_.IncomingCapturedFrame(
8580 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8581 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8582 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8583
8584 video_stream_encoder_->Stop();
8585}
8586
Henrik Boström56db9ff2021-03-24 09:06:45 +01008587std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8588 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8589 VideoCodecType codec_type = std::get<0>(info.param);
8590 bool allow_i420_conversion = std::get<1>(info.param);
8591 std::string str;
8592 switch (codec_type) {
8593 case kVideoCodecGeneric:
8594 str = "Generic";
8595 break;
8596 case kVideoCodecVP8:
8597 str = "VP8";
8598 break;
8599 case kVideoCodecVP9:
8600 str = "VP9";
8601 break;
8602 case kVideoCodecAV1:
8603 str = "AV1";
8604 break;
8605 case kVideoCodecH264:
8606 str = "H264";
8607 break;
8608 case kVideoCodecMultiplex:
8609 str = "Multiplex";
8610 break;
8611 default:
8612 RTC_NOTREACHED();
8613 }
8614 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8615 return str;
8616}
8617
8618constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8619 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8620constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8621 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8622constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8623 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8624constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8625 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8626#if defined(WEBRTC_USE_H264)
8627constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8628 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8629
8630// The windows compiler does not tolerate #if statements inside the
8631// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8632// and without H264).
8633INSTANTIATE_TEST_SUITE_P(
8634 All,
8635 VideoStreamEncoderWithRealEncoderTest,
8636 ::testing::Values(kVP8DisallowConversion,
8637 kVP9DisallowConversion,
8638 kAV1AllowConversion,
8639 kMultiplexDisallowConversion,
8640 kH264AllowConversion),
8641 TestParametersVideoCodecAndAllowI420ConversionToString);
8642#else
8643INSTANTIATE_TEST_SUITE_P(
8644 All,
8645 VideoStreamEncoderWithRealEncoderTest,
8646 ::testing::Values(kVP8DisallowConversion,
8647 kVP9DisallowConversion,
8648 kAV1AllowConversion,
8649 kMultiplexDisallowConversion),
8650 TestParametersVideoCodecAndAllowI420ConversionToString);
8651#endif
8652
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008653class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8654 protected:
8655 void RunTest(const std::vector<VideoStream>& configs,
8656 const int expected_num_init_encode) {
8657 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008658 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008659 InsertFrameAndWaitForEncoded();
8660 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8661 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008662 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8663 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008664
8665 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8666 ConfigureEncoder(configs[1]);
8667 InsertFrameAndWaitForEncoded();
8668 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8669 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008670 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008671 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008672 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008673
8674 video_stream_encoder_->Stop();
8675 }
8676
8677 void ConfigureEncoder(const VideoStream& stream) {
8678 VideoEncoderConfig config;
8679 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8680 config.max_bitrate_bps = stream.max_bitrate_bps;
8681 config.simulcast_layers[0] = stream;
8682 config.video_stream_factory =
8683 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8684 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8685 /*conference_mode=*/false);
8686 video_stream_encoder_->ConfigureEncoder(std::move(config),
8687 kMaxPayloadLength);
8688 }
8689
8690 void OnBitrateUpdated(DataRate bitrate) {
8691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8692 bitrate, bitrate, bitrate, 0, 0, 0);
8693 }
8694
8695 void InsertFrameAndWaitForEncoded() {
8696 timestamp_ms_ += kFrameIntervalMs;
8697 video_source_.IncomingCapturedFrame(
8698 CreateFrame(timestamp_ms_, kWidth, kHeight));
8699 sink_.WaitForEncodedFrame(timestamp_ms_);
8700 }
8701
8702 void ExpectEqual(const VideoCodec& actual,
8703 const VideoStream& expected) const {
8704 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8705 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8706 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8707 static_cast<unsigned int>(expected.min_bitrate_bps));
8708 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8709 static_cast<unsigned int>(expected.max_bitrate_bps));
8710 EXPECT_EQ(actual.simulcastStream[0].width,
8711 kWidth / expected.scale_resolution_down_by);
8712 EXPECT_EQ(actual.simulcastStream[0].height,
8713 kHeight / expected.scale_resolution_down_by);
8714 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8715 expected.num_temporal_layers);
8716 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8717 }
8718
8719 VideoStream DefaultConfig() const {
8720 VideoStream stream;
8721 stream.max_framerate = 25;
8722 stream.min_bitrate_bps = 35000;
8723 stream.max_bitrate_bps = 900000;
8724 stream.scale_resolution_down_by = 1.0;
8725 stream.num_temporal_layers = 1;
8726 stream.bitrate_priority = 1.0;
8727 stream.scalability_mode = "";
8728 return stream;
8729 }
8730
8731 const int kWidth = 640;
8732 const int kHeight = 360;
8733 int64_t timestamp_ms_ = 0;
8734};
8735
8736TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8737 VideoStream config1 = DefaultConfig();
8738 VideoStream config2 = config1;
8739 config2.max_framerate++;
8740
8741 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8742}
8743
8744TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8745 VideoStream config1 = DefaultConfig();
8746 VideoStream config2 = config1;
8747 config2.min_bitrate_bps += 10000;
8748
8749 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8750}
8751
8752TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8753 VideoStream config1 = DefaultConfig();
8754 VideoStream config2 = config1;
8755 config2.max_bitrate_bps += 100000;
8756
8757 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8758}
8759
8760TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8761 VideoStream config1 = DefaultConfig();
8762 VideoStream config2 = config1;
8763 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8764
8765 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8766}
8767
8768TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8769 VideoStream config1 = DefaultConfig();
8770 VideoStream config2 = config1;
8771 config2.scale_resolution_down_by *= 2;
8772
8773 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8774}
8775
8776TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8777 VideoStream config1 = DefaultConfig();
8778 VideoStream config2 = config1;
8779 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8780
8781 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8782}
8783
8784TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8785 VideoStream config1 = DefaultConfig();
8786 VideoStream config2 = config1;
8787 config2.scalability_mode = "L1T2";
8788
8789 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8790}
8791
Markus Handellb4e96d42021-11-05 12:00:55 +01008792TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8793 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8794 auto* adapter_ptr = adapter.get();
8795 SimpleVideoStreamEncoderFactory factory;
8796 auto video_stream_encoder = factory.Create(std::move(adapter));
8797
8798 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true));
8799 VideoEncoderConfig config;
8800 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8801 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
8802 Mock::VerifyAndClearExpectations(adapter_ptr);
8803
8804 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false));
8805 VideoEncoderConfig config2;
8806 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8807 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
8808}
8809
8810TEST(VideoStreamEncoderFrameCadenceTest,
8811 ForwardsFramesIntoFrameCadenceAdapter) {
8812 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8813 auto* adapter_ptr = adapter.get();
8814 test::FrameForwarder video_source;
8815 SimpleVideoStreamEncoderFactory factory;
8816 auto video_stream_encoder = factory.Create(std::move(adapter));
8817 video_stream_encoder->SetSource(
8818 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8819
8820 EXPECT_CALL(*adapter_ptr, OnFrame);
8821 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8822 video_source.IncomingCapturedFrame(
8823 VideoFrame::Builder()
8824 .set_video_frame_buffer(std::move(buffer))
8825 .set_ntp_time_ms(0)
8826 .set_timestamp_ms(0)
8827 .set_rotation(kVideoRotation_0)
8828 .build());
8829}
8830
perkj26091b12016-09-01 01:17:40 -07008831} // namespace webrtc