blob: 3baa9c565d4a844b9aef8eb626baa50c9aea0ed7 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Markus Handelle1a198b2022-09-15 08:13:25 +000055#include "modules/video_coding/utility/vp8_constants.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020056#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010057#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020058#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080060#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020061#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020062#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/encoder_settings.h"
64#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010065#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020066#include "test/gmock.h"
67#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010068#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010069#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020070#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090071#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020072#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010073#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020074#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070075
76namespace webrtc {
77
sprang57c2fff2017-01-16 06:24:02 -080078using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020079using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020080using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020081using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020082using ::testing::Ge;
83using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010084using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020085using ::testing::Le;
86using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010087using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010088using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010089using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010090using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010091using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010092using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020093using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080094
perkj803d97f2016-11-01 11:45:46 -070095namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020096const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010097const int kQpLow = 1;
98const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020099const int kMinFramerateFps = 2;
100const int kMinBalancedFramerateFps = 7;
Markus Handell2cfc1af2022-08-19 08:16:48 +0000101constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
asapersson5f7226f2016-11-25 04:37:00 -0800102const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200103const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
104const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
105const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
106const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800107const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700108const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200109const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200110const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200111const VideoEncoder::ResolutionBitrateLimits
112 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
113const VideoEncoder::ResolutionBitrateLimits
114 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800115
Asa Persson606d3cb2021-10-04 10:07:11 +0200116uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200117 0x00, 0x00, 0x03, 0x03, 0xF4,
118 0x05, 0x03, 0xC7, 0xE0, 0x1B,
119 0x41, 0x10, 0x8D, 0x00};
120
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100121const uint8_t kCodedFrameVp8Qp25[] = {
122 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
123 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
124 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
125
Markus Handell818e7fb2021-12-30 13:01:33 +0100126VideoFrame CreateSimpleNV12Frame() {
127 return VideoFrame::Builder()
128 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
129 /*width=*/16, /*height=*/16))
130 .build();
131}
132
Markus Handell8d87c462021-12-16 11:37:16 +0100133void PassAFrame(
134 TaskQueueBase* encoder_queue,
135 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
136 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200137 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
138 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
139 CreateSimpleNV12Frame());
140 });
Markus Handell8d87c462021-12-16 11:37:16 +0100141}
142
perkj803d97f2016-11-01 11:45:46 -0700143class TestBuffer : public webrtc::I420Buffer {
144 public:
145 TestBuffer(rtc::Event* event, int width, int height)
146 : I420Buffer(width, height), event_(event) {}
147
148 private:
149 friend class rtc::RefCountedObject<TestBuffer>;
150 ~TestBuffer() override {
151 if (event_)
152 event_->Set();
153 }
154 rtc::Event* const event_;
155};
156
Henrik Boström56db9ff2021-03-24 09:06:45 +0100157// A fake native buffer that can't be converted to I420. Upon scaling, it
158// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700159class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
160 public:
161 FakeNativeBuffer(rtc::Event* event, int width, int height)
162 : event_(event), width_(width), height_(height) {}
163 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
164 int width() const override { return width_; }
165 int height() const override { return height_; }
166 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
167 return nullptr;
168 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100169 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
170 int offset_x,
171 int offset_y,
172 int crop_width,
173 int crop_height,
174 int scaled_width,
175 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200176 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
177 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100178 }
Noah Richards51db4212019-06-12 06:59:12 -0700179
180 private:
181 friend class rtc::RefCountedObject<FakeNativeBuffer>;
182 ~FakeNativeBuffer() override {
183 if (event_)
184 event_->Set();
185 }
186 rtc::Event* const event_;
187 const int width_;
188 const int height_;
189};
190
Evan Shrubsole895556e2020-10-05 09:15:13 +0200191// A fake native buffer that is backed by an NV12 buffer.
192class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
193 public:
194 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
195 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
196
197 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
198 int width() const override { return nv12_buffer_->width(); }
199 int height() const override { return nv12_buffer_->height(); }
200 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
201 return nv12_buffer_->ToI420();
202 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200203 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
204 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
205 if (absl::c_find(types, Type::kNV12) != types.end()) {
206 return nv12_buffer_;
207 }
208 return nullptr;
209 }
Niels Möllerba2de582022-04-20 16:46:26 +0200210 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200211
212 private:
213 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
214 ~FakeNV12NativeBuffer() override {
215 if (event_)
216 event_->Set();
217 }
218 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
219 rtc::Event* const event_;
220};
221
Niels Möller7dc26b72017-12-06 10:27:48 +0100222class CpuOveruseDetectorProxy : public OveruseFrameDetector {
223 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200224 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
225 const FieldTrialsView& field_trials)
226 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200227 last_target_framerate_fps_(-1),
228 framerate_updated_event_(true /* manual_reset */,
229 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100230 virtual ~CpuOveruseDetectorProxy() {}
231
232 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200233 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100234 last_target_framerate_fps_ = framerate_fps;
235 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200236 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100237 }
238
239 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200240 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100241 return last_target_framerate_fps_;
242 }
243
Niels Möller4db138e2018-04-19 09:04:13 +0200244 CpuOveruseOptions GetOptions() { return options_; }
245
Henrik Boström381d1092020-05-12 18:49:07 +0200246 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
247
Niels Möller7dc26b72017-12-06 10:27:48 +0100248 private:
Markus Handella3765182020-07-08 13:13:32 +0200249 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100250 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200251 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100252};
253
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254class FakeVideoSourceRestrictionsListener
255 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200256 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200257 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200258 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200259 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200260 RTC_DCHECK(was_restrictions_updated_);
261 }
262
263 rtc::Event* restrictions_updated_event() {
264 return &restrictions_updated_event_;
265 }
266
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200267 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200268 void OnVideoSourceRestrictionsUpdated(
269 VideoSourceRestrictions restrictions,
270 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200271 rtc::scoped_refptr<Resource> reason,
272 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200273 was_restrictions_updated_ = true;
274 restrictions_updated_event_.Set();
275 }
276
277 private:
278 bool was_restrictions_updated_;
279 rtc::Event restrictions_updated_event_;
280};
281
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200282auto WantsFps(Matcher<int> fps_matcher) {
283 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
284 fps_matcher);
285}
286
287auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
288 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
289 AllOf(max_pixel_matcher, Gt(0)));
290}
291
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200292auto ResolutionMax() {
293 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200294 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200295 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
296 Eq(absl::nullopt)));
297}
298
299auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200300 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200301}
302
303auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200304 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200305}
306
307auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200308 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200309}
310
311auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200312 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200313}
314
315auto FpsMaxResolutionMax() {
316 return AllOf(FpsMax(), ResolutionMax());
317}
318
319auto UnlimitedSinkWants() {
320 return AllOf(FpsUnlimited(), ResolutionMax());
321}
322
323auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
324 Matcher<int> fps_range_matcher;
325
326 if (last_frame_pixels <= 320 * 240) {
327 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200328 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200329 fps_range_matcher = AllOf(Ge(10), Le(15));
330 } else if (last_frame_pixels <= 640 * 480) {
331 fps_range_matcher = Ge(15);
332 } else {
333 fps_range_matcher = Eq(kDefaultFramerate);
334 }
335 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
336 fps_range_matcher);
337}
338
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200339auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
340 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
341 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
342}
343
344auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
346}
347
348auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
349 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
350}
351
352auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
353 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
354 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
355}
356
357auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
358 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
359 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
360}
361
362auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
363 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
364 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
365}
366
367auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
368 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
369 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
370}
371
mflodmancc3d4422017-08-03 08:27:51 -0700372class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700373 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100374 VideoStreamEncoderUnderTest(
375 TimeController* time_controller,
376 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
377 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
378 encoder_queue,
379 SendStatisticsProxy* stats_proxy,
380 const VideoStreamEncoderSettings& settings,
381 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100382 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200383 const FieldTrialsView& field_trials,
384 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200385 : VideoStreamEncoder(
386 time_controller->GetClock(),
387 num_cores,
388 stats_proxy,
389 settings,
390 std::unique_ptr<OveruseFrameDetector>(
391 overuse_detector_proxy_ =
392 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
393 std::move(cadence_adapter),
394 std::move(encoder_queue),
395 allocation_callback_type,
396 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200397 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200398 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200399 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200400 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200401 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200402 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200403 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100405 }
perkj803d97f2016-11-01 11:45:46 -0700406
Henrik Boström381d1092020-05-12 18:49:07 +0200407 void SetSourceAndWaitForRestrictionsUpdated(
408 rtc::VideoSourceInterface<VideoFrame>* source,
409 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200410 FakeVideoSourceRestrictionsListener listener;
411 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200412 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000413 listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200414 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200415 }
416
417 void SetSourceAndWaitForFramerateUpdated(
418 rtc::VideoSourceInterface<VideoFrame>* source,
419 const DegradationPreference& degradation_preference) {
420 overuse_detector_proxy_->framerate_updated_event()->Reset();
421 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000422 overuse_detector_proxy_->framerate_updated_event()->Wait(
423 TimeDelta::Seconds(5));
Henrik Boström381d1092020-05-12 18:49:07 +0200424 }
425
426 void OnBitrateUpdatedAndWaitForManagedResources(
427 DataRate target_bitrate,
428 DataRate stable_target_bitrate,
429 DataRate link_allocation,
430 uint8_t fraction_lost,
431 int64_t round_trip_time_ms,
432 double cwnd_reduce_ratio) {
433 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
434 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
435 // Bitrate is updated on the encoder queue.
436 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200437 }
438
kthelgason2fc52542017-03-03 00:24:41 -0800439 // This is used as a synchronisation mechanism, to make sure that the
440 // encoder queue is not blocked before we start sending it frames.
441 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100442 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800443 }
444
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200446 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200448 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200449 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200450 event.Set();
451 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000452 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100453 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200454 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200455
Henrik Boström91aa7322020-04-28 12:24:33 +0200456 void TriggerCpuUnderuse() {
457 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200458 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200459 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200460 event.Set();
461 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000462 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100463 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200464 }
kthelgason876222f2016-11-29 01:44:11 -0800465
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200467 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200468 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200469 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200470 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200471 event.Set();
472 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000473 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100474 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200475 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200476 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200477 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200478 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200479 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200480 event.Set();
481 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000482 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100483 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200484 }
485
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200486 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100487 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200488 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
489 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200490 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700491};
492
Noah Richards51db4212019-06-12 06:59:12 -0700493// Simulates simulcast behavior and makes highest stream resolutions divisible
494// by 4.
495class CroppingVideoStreamFactory
496 : public VideoEncoderConfig::VideoStreamFactoryInterface {
497 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200498 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700499
500 private:
501 std::vector<VideoStream> CreateEncoderStreams(
502 int width,
503 int height,
504 const VideoEncoderConfig& encoder_config) override {
505 std::vector<VideoStream> streams = test::CreateVideoStreams(
506 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700507 return streams;
508 }
Noah Richards51db4212019-06-12 06:59:12 -0700509};
510
sprangb1ca0732017-02-01 08:38:12 -0800511class AdaptingFrameForwarder : public test::FrameForwarder {
512 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200513 explicit AdaptingFrameForwarder(TimeController* time_controller)
514 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700515 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800516
517 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200518 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800519 adaptation_enabled_ = enabled;
520 }
521
asaperssonfab67072017-04-04 05:51:49 -0700522 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200523 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800524 return adaptation_enabled_;
525 }
526
Henrik Boström1124ed12021-02-25 10:30:39 +0100527 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
528 // the resolution or frame rate was different than it is currently. If
529 // something else is modified, such as encoder resolutions, but the resolution
530 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700531 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200532 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700533 return last_wants_;
534 }
535
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200536 absl::optional<int> last_sent_width() const { return last_width_; }
537 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800538
sprangb1ca0732017-02-01 08:38:12 -0800539 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200540 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100541 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542
sprangb1ca0732017-02-01 08:38:12 -0800543 int cropped_width = 0;
544 int cropped_height = 0;
545 int out_width = 0;
546 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700547 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000548 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
549 << "w=" << video_frame.width()
550 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700551 if (adapter_.AdaptFrameResolution(
552 video_frame.width(), video_frame.height(),
553 video_frame.timestamp_us() * 1000, &cropped_width,
554 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100555 VideoFrame adapted_frame =
556 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200557 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100558 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200559 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100560 .set_timestamp_ms(99)
561 .set_rotation(kVideoRotation_0)
562 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100563 if (video_frame.has_update_rect()) {
564 adapted_frame.set_update_rect(
565 video_frame.update_rect().ScaleWithFrame(
566 video_frame.width(), video_frame.height(), 0, 0,
567 video_frame.width(), video_frame.height(), out_width,
568 out_height));
569 }
sprangc5d62e22017-04-02 23:53:04 -0700570 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800571 last_width_.emplace(adapted_frame.width());
572 last_height_.emplace(adapted_frame.height());
573 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200574 last_width_ = absl::nullopt;
575 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700576 }
sprangb1ca0732017-02-01 08:38:12 -0800577 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000578 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800579 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800580 last_width_.emplace(video_frame.width());
581 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800582 }
583 }
584
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200585 void OnOutputFormatRequest(int width, int height) {
586 absl::optional<std::pair<int, int>> target_aspect_ratio =
587 std::make_pair(width, height);
588 absl::optional<int> max_pixel_count = width * height;
589 absl::optional<int> max_fps;
590 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
591 max_fps);
592 }
593
sprangb1ca0732017-02-01 08:38:12 -0800594 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
595 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200596 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100597 rtc::VideoSinkWants prev_wants = sink_wants_locked();
598 bool did_adapt =
599 prev_wants.max_pixel_count != wants.max_pixel_count ||
600 prev_wants.target_pixel_count != wants.target_pixel_count ||
601 prev_wants.max_framerate_fps != wants.max_framerate_fps;
602 if (did_adapt) {
603 last_wants_ = prev_wants;
604 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100605 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200606 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800607 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200608
Erik Språng5e13d052022-08-02 11:42:49 +0200609 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
610
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200611 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800612 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200613 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
614 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200615 absl::optional<int> last_width_;
616 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200617 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800618};
sprangc5d62e22017-04-02 23:53:04 -0700619
Niels Möller213618e2018-07-24 09:29:58 +0200620// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700621class MockableSendStatisticsProxy : public SendStatisticsProxy {
622 public:
623 MockableSendStatisticsProxy(Clock* clock,
624 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100625 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200626 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100627 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700628
629 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200630 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700631 if (mock_stats_)
632 return *mock_stats_;
633 return SendStatisticsProxy::GetStats();
634 }
635
Niels Möller213618e2018-07-24 09:29:58 +0200636 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200637 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200638 if (mock_stats_)
639 return mock_stats_->input_frame_rate;
640 return SendStatisticsProxy::GetInputFrameRate();
641 }
sprangc5d62e22017-04-02 23:53:04 -0700642 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200643 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700644 mock_stats_.emplace(stats);
645 }
646
647 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200648 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700649 mock_stats_.reset();
650 }
651
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200652 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
653 on_frame_dropped_ = std::move(callback);
654 }
655
sprangc5d62e22017-04-02 23:53:04 -0700656 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200657 void OnFrameDropped(DropReason reason) override {
658 SendStatisticsProxy::OnFrameDropped(reason);
659 if (on_frame_dropped_)
660 on_frame_dropped_(reason);
661 }
662
Markus Handella3765182020-07-08 13:13:32 +0200663 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200664 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200665 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700666};
667
Markus Handellb4e96d42021-11-05 12:00:55 +0100668class SimpleVideoStreamEncoderFactory {
669 public:
670 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
671 public:
672 using VideoStreamEncoder::VideoStreamEncoder;
673 ~AdaptedVideoStreamEncoder() { Stop(); }
674 };
675
Markus Handell8d87c462021-12-16 11:37:16 +0100676 class MockFakeEncoder : public test::FakeEncoder {
677 public:
678 using FakeEncoder::FakeEncoder;
679 MOCK_METHOD(CodecSpecificInfo,
680 EncodeHook,
681 (EncodedImage & encoded_image,
682 rtc::scoped_refptr<EncodedImageBuffer> buffer),
683 (override));
684 };
685
Markus Handellee225432021-11-29 12:35:12 +0100686 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100687 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100688 encoder_settings_.bitrate_allocator_factory =
689 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100690 }
691
Markus Handell818e7fb2021-12-30 13:01:33 +0100692 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100693 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100694 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200695 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100696 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
697 time_controller_.GetClock(),
698 /*number_of_cores=*/1,
699 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200700 std::make_unique<CpuOveruseDetectorProxy>(
701 /*stats_proxy=*/nullptr,
702 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100703 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100704 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100705 kVideoBitrateAllocation,
706 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100707 result->SetSink(&sink_, /*rotation_applied=*/false);
708 return result;
709 }
710
Markus Handell818e7fb2021-12-30 13:01:33 +0100711 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
712 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
713 TaskQueueBase** encoder_queue_ptr = nullptr) {
714 auto encoder_queue =
715 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
716 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
717 if (encoder_queue_ptr)
718 *encoder_queue_ptr = encoder_queue.get();
719 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
720 std::move(encoder_queue));
721 }
722
Markus Handell9a478b52021-11-18 16:07:01 +0100723 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100724 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100725
Markus Handell818e7fb2021-12-30 13:01:33 +0100726 GlobalSimulatedTimeController* GetTimeController() {
727 return &time_controller_;
728 }
729
Markus Handellb4e96d42021-11-05 12:00:55 +0100730 private:
731 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
732 public:
733 ~NullEncoderSink() override = default;
734 void OnEncoderConfigurationChanged(
735 std::vector<VideoStream> streams,
736 bool is_svc,
737 VideoEncoderConfig::ContentType content_type,
738 int min_transmit_bitrate_bps) override {}
739 void OnBitrateAllocationUpdated(
740 const VideoBitrateAllocation& allocation) override {}
741 void OnVideoLayersAllocationUpdated(
742 VideoLayersAllocation allocation) override {}
743 Result OnEncodedImage(
744 const EncodedImage& encoded_image,
745 const CodecSpecificInfo* codec_specific_info) override {
746 return Result(EncodedImageCallback::Result::OK);
747 }
748 };
749
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100750 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200751 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100752 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
753 time_controller_.CreateTaskQueueFactory()};
754 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
755 std::make_unique<MockableSendStatisticsProxy>(
756 time_controller_.GetClock(),
757 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100758 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
759 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100760 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
761 CreateBuiltinVideoBitrateAllocatorFactory();
762 VideoStreamEncoderSettings encoder_settings_{
763 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100764 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
765 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100766 NullEncoderSink sink_;
767};
768
769class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
770 public:
771 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100772 MOCK_METHOD(void,
773 SetZeroHertzModeEnabled,
774 (absl::optional<ZeroHertzModeParams>),
775 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100776 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100777 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
778 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100779 MOCK_METHOD(void,
780 UpdateLayerQualityConvergence,
Markus Handell5a77e512022-09-01 12:51:50 +0000781 (size_t spatial_index, bool converged),
Markus Handell8d87c462021-12-16 11:37:16 +0100782 (override));
783 MOCK_METHOD(void,
784 UpdateLayerStatus,
Markus Handell5a77e512022-09-01 12:51:50 +0000785 (size_t spatial_index, bool enabled),
Markus Handell8d87c462021-12-16 11:37:16 +0100786 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100787 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100788};
789
philipel9b058032020-02-10 11:30:00 +0100790class MockEncoderSelector
791 : public VideoEncoderFactory::EncoderSelectorInterface {
792 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200793 MOCK_METHOD(void,
794 OnCurrentEncoder,
795 (const SdpVideoFormat& format),
796 (override));
797 MOCK_METHOD(absl::optional<SdpVideoFormat>,
798 OnAvailableBitrate,
799 (const DataRate& rate),
800 (override));
philipel6daa3042022-04-11 10:48:28 +0200801 MOCK_METHOD(absl::optional<SdpVideoFormat>,
802 OnResolutionChange,
803 (const RenderResolution& resolution),
804 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200805 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100806};
807
Markus Handell2e0f4f02021-12-21 19:14:58 +0100808class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
809 public:
810 MOCK_METHOD(void,
811 AddOrUpdateSink,
812 (rtc::VideoSinkInterface<VideoFrame>*,
813 const rtc::VideoSinkWants&),
814 (override));
815 MOCK_METHOD(void,
816 RemoveSink,
817 (rtc::VideoSinkInterface<VideoFrame>*),
818 (override));
819 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
820};
821
perkj803d97f2016-11-01 11:45:46 -0700822} // namespace
823
mflodmancc3d4422017-08-03 08:27:51 -0700824class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700825 public:
Markus Handell2cfc1af2022-08-19 08:16:48 +0000826 static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
perkj26091b12016-09-01 01:17:40 -0700827
mflodmancc3d4422017-08-03 08:27:51 -0700828 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700829 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700830 codec_width_(320),
831 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200832 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200833 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200834 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700835 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200836 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700837 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100838 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
839 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200840 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700841
842 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700843 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700844 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200845 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800846 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200847 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200848 video_send_config_.rtp.payload_name = "FAKE";
849 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700850
Per512ecb32016-09-23 15:52:06 +0200851 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200852 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200853 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
854 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
855 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100856 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700857
Niels Möllerf1338562018-04-26 09:51:47 +0200858 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800859 }
860
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100861 void ConfigureEncoder(
862 VideoEncoderConfig video_encoder_config,
863 VideoStreamEncoder::BitrateAllocationCallbackType
864 allocation_callback_type =
865 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200866 kVideoBitrateAllocationWhenScreenSharing,
867 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700868 if (video_stream_encoder_)
869 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100870
871 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
872 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
873 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
874 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
875 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100876 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100877 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
878 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
879 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200880 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200881 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700882 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700883 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200884 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700885 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200886 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700887 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800888 }
889
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100890 void ResetEncoder(const std::string& payload_name,
891 size_t num_streams,
892 size_t num_temporal_layers,
893 unsigned char num_spatial_layers,
894 bool screenshare,
895 VideoStreamEncoder::BitrateAllocationCallbackType
896 allocation_callback_type =
897 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200898 kVideoBitrateAllocationWhenScreenSharing,
899 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200900 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800901
902 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200903 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
904 num_streams, &video_encoder_config);
905 for (auto& layer : video_encoder_config.simulcast_layers) {
906 layer.num_temporal_layers = num_temporal_layers;
907 layer.max_framerate = kDefaultFramerate;
908 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100909 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200910 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700911 video_encoder_config.content_type =
912 screenshare ? VideoEncoderConfig::ContentType::kScreen
913 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700914 if (payload_name == "VP9") {
915 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
916 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200917 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700918 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200919 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
920 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700921 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200922 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
923 num_cores);
perkj26091b12016-09-01 01:17:40 -0700924 }
925
sprang57c2fff2017-01-16 06:24:02 -0800926 VideoFrame CreateFrame(int64_t ntp_time_ms,
927 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200928 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200929 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200930 destruction_event, codec_width_, codec_height_))
931 .set_ntp_time_ms(ntp_time_ms)
932 .set_timestamp_ms(99)
933 .set_rotation(kVideoRotation_0)
934 .build();
perkj26091b12016-09-01 01:17:40 -0700935 }
936
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100937 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
938 rtc::Event* destruction_event,
939 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200940 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200941 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200942 destruction_event, codec_width_, codec_height_))
943 .set_ntp_time_ms(ntp_time_ms)
944 .set_timestamp_ms(99)
945 .set_rotation(kVideoRotation_0)
946 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
947 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100948 }
949
sprang57c2fff2017-01-16 06:24:02 -0800950 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200951 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
952 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200953 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200954 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200955 .set_ntp_time_ms(ntp_time_ms)
956 .set_timestamp_ms(ntp_time_ms)
957 .set_rotation(kVideoRotation_0)
958 .build();
perkj803d97f2016-11-01 11:45:46 -0700959 }
960
Evan Shrubsole895556e2020-10-05 09:15:13 +0200961 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200962 return VideoFrame::Builder()
963 .set_video_frame_buffer(NV12Buffer::Create(width, height))
964 .set_ntp_time_ms(ntp_time_ms)
965 .set_timestamp_ms(ntp_time_ms)
966 .set_rotation(kVideoRotation_0)
967 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200968 }
969
Noah Richards51db4212019-06-12 06:59:12 -0700970 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
971 rtc::Event* destruction_event,
972 int width,
973 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200974 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200975 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200976 destruction_event, width, height))
977 .set_ntp_time_ms(ntp_time_ms)
978 .set_timestamp_ms(99)
979 .set_rotation(kVideoRotation_0)
980 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700981 }
982
Evan Shrubsole895556e2020-10-05 09:15:13 +0200983 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
984 rtc::Event* destruction_event,
985 int width,
986 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200987 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200988 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200989 destruction_event, width, height))
990 .set_ntp_time_ms(ntp_time_ms)
991 .set_timestamp_ms(99)
992 .set_rotation(kVideoRotation_0)
993 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200994 }
995
Noah Richards51db4212019-06-12 06:59:12 -0700996 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
997 rtc::Event* destruction_event) const {
998 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
999 codec_height_);
1000 }
1001
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001002 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001004 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001005
1006 video_source_.IncomingCapturedFrame(
1007 CreateFrame(1, codec_width_, codec_height_));
1008 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001009 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001010 }
1011
sprang4847ae62017-06-27 07:06:52 -07001012 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1013 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001014 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001015 }
1016
Markus Handell2cfc1af2022-08-19 08:16:48 +00001017 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
1018 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001019 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001020 return ok;
1021 }
1022
1023 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1024 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001025 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001026 }
1027
1028 void ExpectDroppedFrame() {
1029 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001030 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001031 }
1032
Markus Handell2cfc1af2022-08-19 08:16:48 +00001033 bool WaitForFrame(TimeDelta timeout) {
1034 bool ok = sink_.WaitForFrame(timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001035 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001036 return ok;
1037 }
1038
perkj26091b12016-09-01 01:17:40 -07001039 class TestEncoder : public test::FakeEncoder {
1040 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001041 explicit TestEncoder(TimeController* time_controller)
1042 : FakeEncoder(time_controller->GetClock()),
1043 time_controller_(time_controller) {
1044 RTC_DCHECK(time_controller_);
1045 }
perkj26091b12016-09-01 01:17:40 -07001046
Erik Språngaed30702018-11-05 12:57:17 +01001047 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001048 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001049 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001050 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001051 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001052 info.scaling_settings = VideoEncoder::ScalingSettings(
1053 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001054 }
1055 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001056 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1057 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001058 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001059 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001060 for (int tid = 0; tid < num_layers; ++tid)
1061 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001062 }
1063 }
Erik Språngaed30702018-11-05 12:57:17 +01001064 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001065
1066 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001067 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001068 info.apply_alignment_to_all_simulcast_layers =
1069 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001070 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001071 if (is_qp_trusted_.has_value()) {
1072 info.is_qp_trusted = is_qp_trusted_;
1073 }
Erik Språngaed30702018-11-05 12:57:17 +01001074 return info;
kthelgason876222f2016-11-29 01:44:11 -08001075 }
1076
Erik Språngb7cb7b52019-02-26 15:52:33 +01001077 int32_t RegisterEncodeCompleteCallback(
1078 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001079 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001080 encoded_image_callback_ = callback;
1081 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1082 }
1083
perkjfa10b552016-10-02 23:45:26 -07001084 void ContinueEncode() { continue_encode_event_.Set(); }
1085
1086 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1087 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001088 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001089 EXPECT_EQ(timestamp_, timestamp);
1090 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1091 }
1092
kthelgason2fc52542017-03-03 00:24:41 -08001093 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001094 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001095 quality_scaling_ = b;
1096 }
kthelgasonad9010c2017-02-14 00:46:51 -08001097
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001098 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001099 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001100 requested_resolution_alignment_ = requested_resolution_alignment;
1101 }
1102
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001103 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1104 MutexLock lock(&local_mutex_);
1105 apply_alignment_to_all_simulcast_layers_ = b;
1106 }
1107
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001108 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001109 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001110 is_hardware_accelerated_ = is_hardware_accelerated;
1111 }
1112
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001113 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1114 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001115 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001116 temporal_layers_supported_[spatial_idx] = supported;
1117 }
1118
Sergey Silkin6456e352019-07-08 17:56:40 +02001119 void SetResolutionBitrateLimits(
1120 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001122 resolution_bitrate_limits_ = thresholds;
1123 }
1124
sprangfe627f32017-03-29 08:24:59 -07001125 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001126 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001127 force_init_encode_failed_ = force_failure;
1128 }
1129
Niels Möller6bb5ab92019-01-11 11:11:10 +01001130 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001131 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001132 rate_factor_ = rate_factor;
1133 }
1134
Erik Språngd7329ca2019-02-21 21:19:53 +01001135 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001136 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001137 return last_framerate_;
1138 }
1139
Erik Språngd7329ca2019-02-21 21:19:53 +01001140 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001141 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001142 return last_update_rect_;
1143 }
1144
Niels Möller87e2d782019-03-07 10:18:23 +01001145 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001146 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001147 return last_frame_types_;
1148 }
1149
1150 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001151 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001152 keyframe ? VideoFrameType::kVideoFrameKey
1153 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001154 {
Markus Handella3765182020-07-08 13:13:32 +02001155 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001156 last_frame_types_ = frame_type;
1157 }
Niels Möllerb859b322019-03-07 12:40:01 +01001158 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001159 }
1160
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001161 void InjectEncodedImage(const EncodedImage& image,
1162 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001163 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001164 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001165 }
1166
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001167 void SetEncodedImageData(
1168 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001169 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001170 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001171 }
1172
Erik Språngd7329ca2019-02-21 21:19:53 +01001173 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001174 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001175 expect_null_frame_ = true;
1176 }
1177
Erik Språng5056af02019-09-02 15:53:11 +02001178 absl::optional<VideoEncoder::RateControlParameters>
1179 GetAndResetLastRateControlSettings() {
1180 auto settings = last_rate_control_settings_;
1181 last_rate_control_settings_.reset();
1182 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001183 }
1184
Henrik Boström56db9ff2021-03-24 09:06:45 +01001185 int GetLastInputWidth() const {
1186 MutexLock lock(&local_mutex_);
1187 return last_input_width_;
1188 }
1189
1190 int GetLastInputHeight() const {
1191 MutexLock lock(&local_mutex_);
1192 return last_input_height_;
1193 }
1194
Evan Shrubsole895556e2020-10-05 09:15:13 +02001195 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1196 MutexLock lock(&local_mutex_);
1197 return last_input_pixel_format_;
1198 }
1199
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001200 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001201 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001202 return num_set_rates_;
1203 }
1204
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001205 void SetPreferredPixelFormats(
1206 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1207 pixel_formats) {
1208 MutexLock lock(&local_mutex_);
1209 preferred_pixel_formats_ = std::move(pixel_formats);
1210 }
1211
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001212 void SetIsQpTrusted(absl::optional<bool> trusted) {
1213 MutexLock lock(&local_mutex_);
1214 is_qp_trusted_ = trusted;
1215 }
1216
Erik Språnge4589cb2022-04-06 16:44:30 +02001217 VideoCodecComplexity LastEncoderComplexity() {
1218 MutexLock lock(&local_mutex_);
1219 return last_encoder_complexity_;
1220 }
1221
perkjfa10b552016-10-02 23:45:26 -07001222 private:
perkj26091b12016-09-01 01:17:40 -07001223 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001224 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001225 {
Markus Handella3765182020-07-08 13:13:32 +02001226 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001227 if (expect_null_frame_) {
1228 EXPECT_EQ(input_image.timestamp(), 0u);
1229 EXPECT_EQ(input_image.width(), 1);
1230 last_frame_types_ = *frame_types;
1231 expect_null_frame_ = false;
1232 } else {
1233 EXPECT_GT(input_image.timestamp(), timestamp_);
1234 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1235 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1236 }
perkj26091b12016-09-01 01:17:40 -07001237
1238 timestamp_ = input_image.timestamp();
1239 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001240 last_input_width_ = input_image.width();
1241 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001242 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001243 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001244 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001245 }
Niels Möllerb859b322019-03-07 12:40:01 +01001246 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001247 return result;
1248 }
1249
Niels Möller08ae7ce2020-09-23 15:58:12 +02001250 CodecSpecificInfo EncodeHook(
1251 EncodedImage& encoded_image,
1252 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001253 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001254 {
1255 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001256 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001257 }
1258 MutexLock lock(&local_mutex_);
1259 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001260 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001261 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001262 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001263 }
1264
sprangfe627f32017-03-29 08:24:59 -07001265 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001266 const Settings& settings) override {
1267 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001268
Markus Handella3765182020-07-08 13:13:32 +02001269 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001270 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001271
Erik Språng82fad3d2018-03-21 09:57:23 +01001272 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001273 // Simulate setting up temporal layers, in order to validate the life
1274 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001275 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001276 frame_buffer_controller_ =
1277 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001278 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001279
1280 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1281
Erik Språngb7cb7b52019-02-26 15:52:33 +01001282 if (force_init_encode_failed_) {
1283 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001284 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001285 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001286
Erik Språngb7cb7b52019-02-26 15:52:33 +01001287 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001288 return res;
1289 }
1290
Erik Språngb7cb7b52019-02-26 15:52:33 +01001291 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001292 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001293 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1294 initialized_ = EncoderState::kUninitialized;
1295 return FakeEncoder::Release();
1296 }
1297
Erik Språng16cb8f52019-04-12 13:59:09 +02001298 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001299 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001300 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001301 VideoBitrateAllocation adjusted_rate_allocation;
1302 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1303 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001304 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001305 adjusted_rate_allocation.SetBitrate(
1306 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001307 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001308 rate_factor_));
1309 }
1310 }
1311 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001312 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001313 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001314 RateControlParameters adjusted_paramters = parameters;
1315 adjusted_paramters.bitrate = adjusted_rate_allocation;
1316 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001317 }
1318
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001319 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001320 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001321 enum class EncoderState {
1322 kUninitialized,
1323 kInitializationFailed,
1324 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001325 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001326 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001327 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1328 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1329 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1330 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1331 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1332 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001333 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1334 false;
Markus Handella3765182020-07-08 13:13:32 +02001335 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001336 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1337 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001338 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001339 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001340 absl::optional<bool>
1341 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001342 local_mutex_);
1343 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1344 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1345 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001346 absl::optional<VideoEncoder::RateControlParameters>
1347 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001348 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1349 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001350 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001351 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001352 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1353 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001354 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001355 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001356 RTC_GUARDED_BY(local_mutex_);
1357 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001358 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1359 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001360 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1361 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001362 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001363 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1364 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001365 };
1366
mflodmancc3d4422017-08-03 08:27:51 -07001367 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001368 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001369 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1370 : time_controller_(time_controller), test_encoder_(test_encoder) {
1371 RTC_DCHECK(time_controller_);
1372 }
perkj26091b12016-09-01 01:17:40 -07001373
perkj26091b12016-09-01 01:17:40 -07001374 void WaitForEncodedFrame(int64_t expected_ntp_time) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001375 EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
sprang4847ae62017-06-27 07:06:52 -07001376 }
1377
1378 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
Markus Handell2cfc1af2022-08-19 08:16:48 +00001379 TimeDelta timeout) {
perkj26091b12016-09-01 01:17:40 -07001380 uint32_t timestamp = 0;
Markus Handell2cfc1af2022-08-19 08:16:48 +00001381 if (!WaitForFrame(timeout))
sprang4847ae62017-06-27 07:06:52 -07001382 return false;
perkj26091b12016-09-01 01:17:40 -07001383 {
Markus Handella3765182020-07-08 13:13:32 +02001384 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001385 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001386 }
1387 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001388 return true;
perkj26091b12016-09-01 01:17:40 -07001389 }
1390
sprangb1ca0732017-02-01 08:38:12 -08001391 void WaitForEncodedFrame(uint32_t expected_width,
1392 uint32_t expected_height) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001393 EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001394 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001395 }
1396
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001397 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001398 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001399 uint32_t width = 0;
1400 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001401 {
Markus Handella3765182020-07-08 13:13:32 +02001402 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001403 width = last_width_;
1404 height = last_height_;
1405 }
1406 EXPECT_EQ(expected_height, height);
1407 EXPECT_EQ(expected_width, width);
1408 }
1409
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001410 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1411 VideoRotation rotation;
1412 {
Markus Handella3765182020-07-08 13:13:32 +02001413 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001414 rotation = last_rotation_;
1415 }
1416 EXPECT_EQ(expected_rotation, rotation);
1417 }
1418
Markus Handell2cfc1af2022-08-19 08:16:48 +00001419 void ExpectDroppedFrame() {
1420 EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
1421 }
kthelgason2bc68642017-02-07 07:02:22 -08001422
Markus Handell2cfc1af2022-08-19 08:16:48 +00001423 bool WaitForFrame(TimeDelta timeout) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001424 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001425 time_controller_->AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001426 bool ret = encoded_frame_event_.Wait(timeout);
Markus Handell28c71802021-11-08 10:11:55 +01001427 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001428 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001429 }
1430
perkj26091b12016-09-01 01:17:40 -07001431 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001432 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001433 expect_frames_ = false;
1434 }
1435
asaperssonfab67072017-04-04 05:51:49 -07001436 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001437 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001438 return number_of_reconfigurations_;
1439 }
1440
asaperssonfab67072017-04-04 05:51:49 -07001441 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001442 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001443 return min_transmit_bitrate_bps_;
1444 }
1445
Erik Språngd7329ca2019-02-21 21:19:53 +01001446 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001447 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001448 num_expected_layers_ = num_layers;
1449 }
1450
Erik Språngb7cb7b52019-02-26 15:52:33 +01001451 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001452 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001453 return last_capture_time_ms_;
1454 }
1455
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001456 const EncodedImage& GetLastEncodedImage() {
1457 MutexLock lock(&mutex_);
1458 return last_encoded_image_;
1459 }
1460
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001461 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001462 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001463 return std::move(last_encoded_image_data_);
1464 }
1465
Per Kjellanderdcef6412020-10-07 15:09:05 +02001466 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1467 MutexLock lock(&mutex_);
1468 return last_bitrate_allocation_;
1469 }
1470
1471 int number_of_bitrate_allocations() const {
1472 MutexLock lock(&mutex_);
1473 return number_of_bitrate_allocations_;
1474 }
1475
Per Kjellandera9434842020-10-15 17:53:22 +02001476 VideoLayersAllocation GetLastVideoLayersAllocation() {
1477 MutexLock lock(&mutex_);
1478 return last_layers_allocation_;
1479 }
1480
1481 int number_of_layers_allocations() const {
1482 MutexLock lock(&mutex_);
1483 return number_of_layers_allocations_;
1484 }
1485
perkj26091b12016-09-01 01:17:40 -07001486 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001487 Result OnEncodedImage(
1488 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001489 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001490 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001491 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001492 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001493 last_encoded_image_data_ = std::vector<uint8_t>(
1494 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001495 uint32_t timestamp = encoded_image.Timestamp();
1496 if (last_timestamp_ != timestamp) {
1497 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001498 last_width_ = encoded_image._encodedWidth;
1499 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001500 } else {
1501 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001502 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1503 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001504 }
1505 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001506 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001507 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001508 if (num_received_layers_ == num_expected_layers_) {
1509 encoded_frame_event_.Set();
1510 }
sprangb1ca0732017-02-01 08:38:12 -08001511 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001512 }
1513
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001514 void OnEncoderConfigurationChanged(
1515 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001516 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001517 VideoEncoderConfig::ContentType content_type,
1518 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001519 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001520 ++number_of_reconfigurations_;
1521 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1522 }
1523
Per Kjellanderdcef6412020-10-07 15:09:05 +02001524 void OnBitrateAllocationUpdated(
1525 const VideoBitrateAllocation& allocation) override {
1526 MutexLock lock(&mutex_);
1527 ++number_of_bitrate_allocations_;
1528 last_bitrate_allocation_ = allocation;
1529 }
1530
Per Kjellandera9434842020-10-15 17:53:22 +02001531 void OnVideoLayersAllocationUpdated(
1532 VideoLayersAllocation allocation) override {
1533 MutexLock lock(&mutex_);
1534 ++number_of_layers_allocations_;
1535 last_layers_allocation_ = allocation;
1536 rtc::StringBuilder log;
1537 for (const auto& layer : allocation.active_spatial_layers) {
1538 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1539 << "[";
1540 for (const auto target_bitrate :
1541 layer.target_bitrate_per_temporal_layer) {
1542 log << target_bitrate.kbps() << ",";
1543 }
1544 log << "]";
1545 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001546 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001547 }
1548
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001549 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001550 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001551 TestEncoder* test_encoder_;
1552 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001553 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001554 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001555 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001556 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001557 uint32_t last_height_ = 0;
1558 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001559 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001560 size_t num_expected_layers_ = 1;
1561 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001562 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001563 int number_of_reconfigurations_ = 0;
1564 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001565 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1566 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001567 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1568 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001569 };
1570
Sergey Silkin5ee69672019-07-02 14:18:34 +02001571 class VideoBitrateAllocatorProxyFactory
1572 : public VideoBitrateAllocatorFactory {
1573 public:
1574 VideoBitrateAllocatorProxyFactory()
1575 : bitrate_allocator_factory_(
1576 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1577
1578 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1579 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001580 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001581 codec_config_ = codec;
1582 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1583 }
1584
1585 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001586 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001587 return codec_config_;
1588 }
1589
1590 private:
1591 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1592
Markus Handella3765182020-07-08 13:13:32 +02001593 mutable Mutex mutex_;
1594 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001595 };
1596
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001597 Clock* clock() { return time_controller_.GetClock(); }
1598 void AdvanceTime(TimeDelta duration) {
1599 time_controller_.AdvanceTime(duration);
1600 }
1601
1602 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1603
1604 protected:
1605 virtual TaskQueueFactory* GetTaskQueueFactory() {
1606 return time_controller_.GetTaskQueueFactory();
1607 }
1608
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001609 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001610 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001611 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001612 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001613 int codec_width_;
1614 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001615 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001616 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001617 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001618 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001619 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001620 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001621 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001622 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001623};
1624
mflodmancc3d4422017-08-03 08:27:51 -07001625TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001626 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001627 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001628 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001629 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001630 WaitForEncodedFrame(1);
Markus Handell2cfc1af2022-08-19 08:16:48 +00001631 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
mflodmancc3d4422017-08-03 08:27:51 -07001632 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001633}
1634
mflodmancc3d4422017-08-03 08:27:51 -07001635TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001636 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001637 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001638 // The encoder will cache up to one frame for a short duration. Adding two
1639 // frames means that the first frame will be dropped and the second frame will
1640 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001641 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001642 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001643 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001644 AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001645 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001646
Henrik Boström381d1092020-05-12 18:49:07 +02001647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001648 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001649
Sebastian Janssona3177052018-04-10 13:05:49 +02001650 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001651 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001652 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1653
1654 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001655 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001656}
1657
mflodmancc3d4422017-08-03 08:27:51 -07001658TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001660 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001661 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001662 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001663
Henrik Boström381d1092020-05-12 18:49:07 +02001664 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001665 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1666
Sebastian Janssona3177052018-04-10 13:05:49 +02001667 // The encoder will cache up to one frame for a short duration. Adding two
1668 // frames means that the first frame will be dropped and the second frame will
1669 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001670 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001671 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001672
Henrik Boström381d1092020-05-12 18:49:07 +02001673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001674 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001675 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001676 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1677 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001678 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001679}
1680
mflodmancc3d4422017-08-03 08:27:51 -07001681TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001683 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001684 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001685 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001686
1687 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001688 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001689
perkja49cbd32016-09-16 07:53:41 -07001690 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001691 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001692 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001693}
1694
mflodmancc3d4422017-08-03 08:27:51 -07001695TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001698
perkja49cbd32016-09-16 07:53:41 -07001699 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001700 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001701
mflodmancc3d4422017-08-03 08:27:51 -07001702 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001703 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001704 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001705 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
Markus Handell2cfc1af2022-08-19 08:16:48 +00001706 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001707}
1708
Markus Handell9a478b52021-11-18 16:07:01 +01001709TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1710 test::FrameForwarder source;
1711 video_stream_encoder_->SetSource(&source,
1712 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001713 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001714 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001715
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001716 int dropped_count = 0;
1717 stats_proxy_->SetDroppedFrameCallback(
1718 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1719 ++dropped_count;
1720 });
1721
Markus Handell9a478b52021-11-18 16:07:01 +01001722 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1723 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1724 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001725 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001726 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001727}
1728
Henrik Boström56db9ff2021-03-24 09:06:45 +01001729TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001731 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001732
1733 rtc::Event frame_destroyed_event;
1734 video_source_.IncomingCapturedFrame(
1735 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001736 WaitForEncodedFrame(1);
1737 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1738 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001739 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1740 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001741 video_stream_encoder_->Stop();
1742}
1743
Henrik Boström56db9ff2021-03-24 09:06:45 +01001744TEST_F(VideoStreamEncoderTest,
1745 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001746 // Use the cropping factory.
1747 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001748 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001749 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1750 kMaxPayloadLength);
1751 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1752
1753 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001754 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001755 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001756 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1757 WaitForEncodedFrame(1);
1758 // The encoder will have been configured once.
1759 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001760 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1761 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001762
1763 // Now send in a fake frame that needs to be cropped as the width/height
1764 // aren't divisible by 4 (see CreateEncoderStreams above).
1765 rtc::Event frame_destroyed_event;
1766 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1767 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001768 WaitForEncodedFrame(2);
1769 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1770 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001771 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1772 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001773 video_stream_encoder_->Stop();
1774}
1775
Evan Shrubsole895556e2020-10-05 09:15:13 +02001776TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1777 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001778 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001779
1780 video_source_.IncomingCapturedFrame(
1781 CreateNV12Frame(1, codec_width_, codec_height_));
1782 WaitForEncodedFrame(1);
1783 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1784 fake_encoder_.GetLastInputPixelFormat());
1785 video_stream_encoder_->Stop();
1786}
1787
Henrik Boström56db9ff2021-03-24 09:06:45 +01001788TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001790 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001791
1792 fake_encoder_.SetPreferredPixelFormats({});
1793
1794 rtc::Event frame_destroyed_event;
1795 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1796 1, &frame_destroyed_event, codec_width_, codec_height_));
1797 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001798 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001799 fake_encoder_.GetLastInputPixelFormat());
1800 video_stream_encoder_->Stop();
1801}
1802
Henrik Boström56db9ff2021-03-24 09:06:45 +01001803TEST_F(VideoStreamEncoderTest,
1804 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001805 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001806 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001807
1808 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1809
1810 rtc::Event frame_destroyed_event;
1811 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1812 1, &frame_destroyed_event, codec_width_, codec_height_));
1813 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001814 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001815 fake_encoder_.GetLastInputPixelFormat());
1816 video_stream_encoder_->Stop();
1817}
1818
Henrik Boström56db9ff2021-03-24 09:06:45 +01001819TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001820 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001821 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001822
1823 // Fake NV12 native frame does not allow mapping to I444.
1824 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1825
1826 rtc::Event frame_destroyed_event;
1827 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1828 1, &frame_destroyed_event, codec_width_, codec_height_));
1829 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001830 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001831 fake_encoder_.GetLastInputPixelFormat());
1832 video_stream_encoder_->Stop();
1833}
1834
Henrik Boström56db9ff2021-03-24 09:06:45 +01001835TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001837 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001838
1839 rtc::Event frame_destroyed_event;
1840 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1841 1, &frame_destroyed_event, codec_width_, codec_height_));
1842 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001843 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001844 fake_encoder_.GetLastInputPixelFormat());
1845 video_stream_encoder_->Stop();
1846}
1847
Ying Wang9b881ab2020-02-07 14:29:32 +01001848TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001850 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001851 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1852 WaitForEncodedFrame(1);
1853
Henrik Boström381d1092020-05-12 18:49:07 +02001854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001855 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001856 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1857 // frames. Adding two frames means that the first frame will be dropped and
1858 // the second frame will be sent to the encoder.
1859 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1860 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1861 WaitForEncodedFrame(3);
1862 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1863 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1864 WaitForEncodedFrame(5);
1865 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1866 video_stream_encoder_->Stop();
1867}
1868
mflodmancc3d4422017-08-03 08:27:51 -07001869TEST_F(VideoStreamEncoderTest,
1870 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001873 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001874
1875 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001876 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001877 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001878 // The encoder will have been configured once when the first frame is
1879 // received.
1880 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001881
1882 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001883 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001884 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001885 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001886 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001887
1888 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001889 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001890 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001891 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001892 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001893
mflodmancc3d4422017-08-03 08:27:51 -07001894 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001895}
1896
mflodmancc3d4422017-08-03 08:27:51 -07001897TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001898 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001899 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001900
1901 // Capture a frame and wait for it to synchronize with the encoder thread.
1902 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001903 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001904 // The encoder will have been configured once.
1905 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001906 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1907 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001908
1909 codec_width_ *= 2;
1910 codec_height_ *= 2;
1911 // Capture a frame with a higher resolution and wait for it to synchronize
1912 // with the encoder thread.
1913 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001914 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001915 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1916 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001917 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001918
mflodmancc3d4422017-08-03 08:27:51 -07001919 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001920}
1921
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001922TEST_F(VideoStreamEncoderTest,
1923 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001926
1927 // Capture a frame and wait for it to synchronize with the encoder thread.
1928 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1929 WaitForEncodedFrame(1);
1930
1931 VideoEncoderConfig video_encoder_config;
1932 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1933 // Changing the max payload data length recreates encoder.
1934 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1935 kMaxPayloadLength / 2);
1936
1937 // Capture a frame and wait for it to synchronize with the encoder thread.
1938 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1939 WaitForEncodedFrame(2);
1940 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1941
1942 video_stream_encoder_->Stop();
1943}
1944
Sergey Silkin5ee69672019-07-02 14:18:34 +02001945TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001946 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001947 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001948
1949 VideoEncoderConfig video_encoder_config;
1950 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001951 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1952 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001953 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1954 kMaxPayloadLength);
1955
1956 // Capture a frame and wait for it to synchronize with the encoder thread.
1957 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1958 WaitForEncodedFrame(1);
1959 // The encoder will have been configured once when the first frame is
1960 // received.
1961 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001962 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001963 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001964 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001965 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1966
Sergey Silkin6456e352019-07-08 17:56:40 +02001967 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1968 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001969 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1970 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001971 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1972 kMaxPayloadLength);
1973
1974 // Capture a frame and wait for it to synchronize with the encoder thread.
1975 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1976 WaitForEncodedFrame(2);
1977 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1978 // Bitrate limits have changed - rate allocator should be reconfigured,
1979 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001980 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001981 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001982 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001983 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001984 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001985
1986 video_stream_encoder_->Stop();
1987}
1988
Sergey Silkin6456e352019-07-08 17:56:40 +02001989TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001990 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001991 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001992 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001993
Sergey Silkincd02eba2020-01-20 14:48:40 +01001994 const uint32_t kMinEncBitrateKbps = 100;
1995 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001996 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001997 /*frame_size_pixels=*/codec_width_ * codec_height_,
1998 /*min_start_bitrate_bps=*/0,
1999 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2000 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002001 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2002
Sergey Silkincd02eba2020-01-20 14:48:40 +01002003 VideoEncoderConfig video_encoder_config;
2004 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2005 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2006 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2007 (kMinEncBitrateKbps + 1) * 1000;
2008 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2009 kMaxPayloadLength);
2010
2011 // When both encoder and app provide bitrate limits, the intersection of
2012 // provided sets should be used.
2013 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2014 WaitForEncodedFrame(1);
2015 EXPECT_EQ(kMaxEncBitrateKbps,
2016 bitrate_allocator_factory_.codec_config().maxBitrate);
2017 EXPECT_EQ(kMinEncBitrateKbps + 1,
2018 bitrate_allocator_factory_.codec_config().minBitrate);
2019
2020 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2021 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2022 (kMinEncBitrateKbps - 1) * 1000;
2023 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2024 kMaxPayloadLength);
2025 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002026 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002027 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002028 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002029 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002030 bitrate_allocator_factory_.codec_config().minBitrate);
2031
Sergey Silkincd02eba2020-01-20 14:48:40 +01002032 video_stream_encoder_->Stop();
2033}
2034
2035TEST_F(VideoStreamEncoderTest,
2036 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002038 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002039
2040 const uint32_t kMinAppBitrateKbps = 100;
2041 const uint32_t kMaxAppBitrateKbps = 200;
2042 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2043 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2044 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2045 /*frame_size_pixels=*/codec_width_ * codec_height_,
2046 /*min_start_bitrate_bps=*/0,
2047 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2048 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2049 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2050
2051 VideoEncoderConfig video_encoder_config;
2052 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2053 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2054 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2055 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2057 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002058
Sergey Silkincd02eba2020-01-20 14:48:40 +01002059 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2060 WaitForEncodedFrame(1);
2061 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002062 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002063 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002065
2066 video_stream_encoder_->Stop();
2067}
2068
2069TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002070 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002072 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002073
2074 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002075 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002076 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002077 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002078 fake_encoder_.SetResolutionBitrateLimits(
2079 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2080
2081 VideoEncoderConfig video_encoder_config;
2082 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2083 video_encoder_config.max_bitrate_bps = 0;
2084 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2085 kMaxPayloadLength);
2086
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002088 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2089 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002090 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2091 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002092 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2093 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2094
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002095 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002096 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2097 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002098 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2099 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002100 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2101 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2102
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002103 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002104 // encoder for 360p should be used.
2105 video_source_.IncomingCapturedFrame(
2106 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2107 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002108 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2109 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002110 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2111 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2112
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002113 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002114 // ignored.
2115 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2116 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002117 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2118 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002119 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2120 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002121 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2122 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002123 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2124 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2125
2126 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2127 // for 270p should be used.
2128 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2129 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002130 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2131 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002132 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2133 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2134
2135 video_stream_encoder_->Stop();
2136}
2137
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002138TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002139 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002140 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002141
2142 VideoEncoderConfig video_encoder_config;
2143 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2144 video_encoder_config.max_bitrate_bps = 0;
2145 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2146 kMaxPayloadLength);
2147
2148 // Encode 720p frame to get the default encoder target bitrate.
2149 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2150 WaitForEncodedFrame(1);
2151 const uint32_t kDefaultTargetBitrateFor720pKbps =
2152 bitrate_allocator_factory_.codec_config()
2153 .simulcastStream[0]
2154 .targetBitrate;
2155
2156 // Set the max recommended encoder bitrate to something lower than the default
2157 // target bitrate.
2158 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2159 1280 * 720, 10 * 1000, 10 * 1000,
2160 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2161 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2162
2163 // Change resolution to trigger encoder reinitialization.
2164 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2165 WaitForEncodedFrame(2);
2166 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2167 WaitForEncodedFrame(3);
2168
2169 // Ensure the target bitrate is capped by the max bitrate.
2170 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2171 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2172 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2173 .simulcastStream[0]
2174 .targetBitrate *
2175 1000,
2176 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2177
2178 video_stream_encoder_->Stop();
2179}
2180
Åsa Perssona7e34d32021-01-20 15:36:13 +01002181TEST_F(VideoStreamEncoderTest,
2182 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2183 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2184 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2185 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2186 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2187 fake_encoder_.SetResolutionBitrateLimits(
2188 {kEncoderLimits270p, kEncoderLimits360p});
2189
2190 // Two streams, highest stream active.
2191 VideoEncoderConfig config;
2192 const int kNumStreams = 2;
2193 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2194 config.max_bitrate_bps = 0;
2195 config.simulcast_layers[0].active = false;
2196 config.simulcast_layers[1].active = true;
2197 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002198 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002199 "VP8", /*max qp*/ 56, /*screencast*/ false,
2200 /*screenshare enabled*/ false);
2201 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2202
2203 // The encoder bitrate limits for 270p should be used.
2204 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002205 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002206 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002207 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002208 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002209 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002210 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002211
2212 // The encoder bitrate limits for 360p should be used.
2213 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002214 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002216 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002217 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002218 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002219
2220 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2221 video_source_.IncomingCapturedFrame(
2222 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002223 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002224 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002225 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002226 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002227 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002228
2229 // Resolution higher than 360p. Encoder limits should be ignored.
2230 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002231 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002232 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002233 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002234 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002235 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002236 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002237 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002238 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002239 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240
2241 // Resolution lower than 270p. The encoder limits for 270p should be used.
2242 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002243 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002244 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002245 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002246 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002248
2249 video_stream_encoder_->Stop();
2250}
2251
2252TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002253 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2254 // Two streams, highest stream active.
2255 VideoEncoderConfig config;
2256 const int kNumStreams = 2;
2257 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2258 config.max_bitrate_bps = 0;
2259 config.simulcast_layers[0].active = false;
2260 config.simulcast_layers[1].active = true;
2261 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002262 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002263 "VP8", /*max qp*/ 56, /*screencast*/ false,
2264 /*screenshare enabled*/ false);
2265 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2266
2267 // Default bitrate limits for 270p should be used.
2268 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2269 kDefaultLimits270p =
2270 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002271 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002272 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002274 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002275 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002276 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002277 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002278 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002279
2280 // Default bitrate limits for 360p should be used.
2281 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2282 kDefaultLimits360p =
2283 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002284 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002285 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002286 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002287 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002288 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002289 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002290 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002291
2292 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2293 video_source_.IncomingCapturedFrame(
2294 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002295 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002296 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002297 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002298 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002299 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002300
2301 // Default bitrate limits for 540p should be used.
2302 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2303 kDefaultLimits540p =
2304 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002305 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002306 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002307 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002308 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002309 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002310 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002311 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002312
2313 video_stream_encoder_->Stop();
2314}
2315
2316TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002317 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2318 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2319 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2320 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2321 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2322 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2323 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2324 fake_encoder_.SetResolutionBitrateLimits(
2325 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2326
2327 // Three streams, middle stream active.
2328 VideoEncoderConfig config;
2329 const int kNumStreams = 3;
2330 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2331 config.simulcast_layers[0].active = false;
2332 config.simulcast_layers[1].active = true;
2333 config.simulcast_layers[2].active = false;
2334 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002335 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002336 "VP8", /*max qp*/ 56, /*screencast*/ false,
2337 /*screenshare enabled*/ false);
2338 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2339
2340 // The encoder bitrate limits for 360p should be used.
2341 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002342 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002343 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002344 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002345 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002346 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002347 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002348
2349 // The encoder bitrate limits for 270p should be used.
2350 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002351 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002352 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002353 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002354 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002355 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002356
2357 video_stream_encoder_->Stop();
2358}
2359
2360TEST_F(VideoStreamEncoderTest,
2361 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2362 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2363 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2364 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2365 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2366 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2367 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2368 fake_encoder_.SetResolutionBitrateLimits(
2369 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2370
2371 // Three streams, lowest stream active.
2372 VideoEncoderConfig config;
2373 const int kNumStreams = 3;
2374 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2375 config.simulcast_layers[0].active = true;
2376 config.simulcast_layers[1].active = false;
2377 config.simulcast_layers[2].active = false;
2378 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002379 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002380 "VP8", /*max qp*/ 56, /*screencast*/ false,
2381 /*screenshare enabled*/ false);
2382 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2383
2384 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2385 // on lowest stream, limits for 270p should not be used
2386 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002387 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002388 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002389 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002390 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002391 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002392 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002393
2394 video_stream_encoder_->Stop();
2395}
2396
2397TEST_F(VideoStreamEncoderTest,
2398 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2399 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2400 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2401 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2402 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2403 fake_encoder_.SetResolutionBitrateLimits(
2404 {kEncoderLimits270p, kEncoderLimits360p});
2405 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2406
2407 // Two streams, highest stream active.
2408 VideoEncoderConfig config;
2409 const int kNumStreams = 2;
2410 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2411 config.simulcast_layers[0].active = false;
2412 config.simulcast_layers[1].active = true;
2413 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2414 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002415 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002416 "VP8", /*max qp*/ 56, /*screencast*/ false,
2417 /*screenshare enabled*/ false);
2418 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2419
2420 // The encoder bitrate limits for 270p should be used.
2421 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002422 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002423 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002424 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002425 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002426 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002427 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002428
2429 // The max configured bitrate is less than the encoder limit for 360p.
2430 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002431 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002432 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002433 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002434 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002435 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002436
2437 video_stream_encoder_->Stop();
2438}
2439
mflodmancc3d4422017-08-03 08:27:51 -07002440TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002441 EXPECT_TRUE(video_source_.has_sinks());
2442 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002443 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002444 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002445 EXPECT_FALSE(video_source_.has_sinks());
2446 EXPECT_TRUE(new_video_source.has_sinks());
2447
mflodmancc3d4422017-08-03 08:27:51 -07002448 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002449}
2450
mflodmancc3d4422017-08-03 08:27:51 -07002451TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002452 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002453 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002454 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002455 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002456}
2457
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002458class ResolutionAlignmentTest
2459 : public VideoStreamEncoderTest,
2460 public ::testing::WithParamInterface<
2461 ::testing::tuple<int, std::vector<double>>> {
2462 public:
2463 ResolutionAlignmentTest()
2464 : requested_alignment_(::testing::get<0>(GetParam())),
2465 scale_factors_(::testing::get<1>(GetParam())) {}
2466
2467 protected:
2468 const int requested_alignment_;
2469 const std::vector<double> scale_factors_;
2470};
2471
2472INSTANTIATE_TEST_SUITE_P(
2473 AlignmentAndScaleFactors,
2474 ResolutionAlignmentTest,
2475 ::testing::Combine(
2476 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2477 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2478 std::vector<double>{-1.0, -1.0},
2479 std::vector<double>{-1.0, -1.0, -1.0},
2480 std::vector<double>{4.0, 2.0, 1.0},
2481 std::vector<double>{9999.0, -1.0, 1.0},
2482 std::vector<double>{3.99, 2.01, 1.0},
2483 std::vector<double>{4.9, 1.7, 1.25},
2484 std::vector<double>{10.0, 4.0, 3.0},
2485 std::vector<double>{1.75, 3.5},
2486 std::vector<double>{1.5, 2.5},
2487 std::vector<double>{1.3, 1.0})));
2488
2489TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2490 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002491 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002492 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2493 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2494
2495 // Fill config with the scaling factor by which to reduce encoding size.
2496 const int num_streams = scale_factors_.size();
2497 VideoEncoderConfig config;
2498 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2499 for (int i = 0; i < num_streams; ++i) {
2500 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2501 }
2502 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002503 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002504 "VP8", /*max qp*/ 56, /*screencast*/ false,
2505 /*screenshare enabled*/ false);
2506 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2507
Henrik Boström381d1092020-05-12 18:49:07 +02002508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002509 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2510 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002511 // Wait for all layers before triggering event.
2512 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002513
2514 // On the 1st frame, we should have initialized the encoder and
2515 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002516 int64_t timestamp_ms = kFrameIntervalMs;
2517 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2518 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002519 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002520
2521 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2522 // (It's up the to the encoder to potentially drop the previous frame,
2523 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002524 timestamp_ms += kFrameIntervalMs;
2525 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2526 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002527 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002528
Asa Persson606d3cb2021-10-04 10:07:11 +02002529 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002530 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2531 // Frame size should be a multiple of the requested alignment.
2532 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2533 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2534 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2535 // Aspect ratio should match.
2536 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2537 codec.height * codec.simulcastStream[i].width);
2538 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002539
2540 video_stream_encoder_->Stop();
2541}
2542
Jonathan Yubc771b72017-12-08 17:04:29 -08002543TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2544 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002545 const int kWidth = 1280;
2546 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002547
2548 // We rely on the automatic resolution adaptation, but we handle framerate
2549 // adaptation manually by mocking the stats proxy.
2550 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002551
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002552 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002553 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002554 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002555 video_stream_encoder_->SetSource(&video_source_,
2556 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002557 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002558 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002560 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2561
Jonathan Yubc771b72017-12-08 17:04:29 -08002562 // Adapt down as far as possible.
2563 rtc::VideoSinkWants last_wants;
2564 int64_t t = 1;
2565 int loop_count = 0;
2566 do {
2567 ++loop_count;
2568 last_wants = video_source_.sink_wants();
2569
2570 // Simulate the framerate we've been asked to adapt to.
2571 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2572 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2573 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2574 mock_stats.input_frame_rate = fps;
2575 stats_proxy_->SetMockStats(mock_stats);
2576
2577 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2578 sink_.WaitForEncodedFrame(t);
2579 t += frame_interval_ms;
2580
mflodmancc3d4422017-08-03 08:27:51 -07002581 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002582 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002583 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002584 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2585 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002586 } while (video_source_.sink_wants().max_pixel_count <
2587 last_wants.max_pixel_count ||
2588 video_source_.sink_wants().max_framerate_fps <
2589 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002590
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 // Verify that we've adapted all the way down.
2592 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002593 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002594 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2595 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002596 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002597 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2598 *video_source_.last_sent_height());
2599 EXPECT_EQ(kMinBalancedFramerateFps,
2600 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002601
Jonathan Yubc771b72017-12-08 17:04:29 -08002602 // Adapt back up the same number of times we adapted down.
2603 for (int i = 0; i < loop_count - 1; ++i) {
2604 last_wants = video_source_.sink_wants();
2605
2606 // Simulate the framerate we've been asked to adapt to.
2607 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2608 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2609 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2610 mock_stats.input_frame_rate = fps;
2611 stats_proxy_->SetMockStats(mock_stats);
2612
2613 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2614 sink_.WaitForEncodedFrame(t);
2615 t += frame_interval_ms;
2616
Henrik Boström91aa7322020-04-28 12:24:33 +02002617 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002618 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002619 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002620 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2621 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002622 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2623 last_wants.max_pixel_count ||
2624 video_source_.sink_wants().max_framerate_fps >
2625 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002626 }
2627
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002628 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002629 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002630 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002631 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2632 EXPECT_EQ((loop_count - 1) * 2,
2633 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002634
mflodmancc3d4422017-08-03 08:27:51 -07002635 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002636}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002637
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002638TEST_F(VideoStreamEncoderTest,
2639 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002640 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2641 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002642 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002643
2644 const int kFrameWidth = 1280;
2645 const int kFrameHeight = 720;
2646
2647 int64_t ntp_time = kFrameIntervalMs;
2648
2649 // Force an input frame rate to be available, or the adaptation call won't
2650 // know what framerate to adapt form.
2651 const int kInputFps = 30;
2652 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2653 stats.input_frame_rate = kInputFps;
2654 stats_proxy_->SetMockStats(stats);
2655
2656 video_source_.set_adaptation_enabled(true);
2657 video_stream_encoder_->SetSource(
2658 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002659 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002660 video_source_.IncomingCapturedFrame(
2661 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2662 sink_.WaitForEncodedFrame(ntp_time);
2663 ntp_time += kFrameIntervalMs;
2664
2665 // Trigger CPU overuse.
2666 video_stream_encoder_->TriggerCpuOveruse();
2667 video_source_.IncomingCapturedFrame(
2668 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2669 sink_.WaitForEncodedFrame(ntp_time);
2670 ntp_time += kFrameIntervalMs;
2671
2672 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2673 EXPECT_EQ(std::numeric_limits<int>::max(),
2674 video_source_.sink_wants().max_pixel_count);
2675 // Some framerate constraint should be set.
2676 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2677 EXPECT_LT(restricted_fps, kInputFps);
2678 video_source_.IncomingCapturedFrame(
2679 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2680 sink_.WaitForEncodedFrame(ntp_time);
2681 ntp_time += 100;
2682
Henrik Boström2671dac2020-05-19 16:29:09 +02002683 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002684 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2685 // Give the encoder queue time to process the change in degradation preference
2686 // by waiting for an encoded frame.
2687 video_source_.IncomingCapturedFrame(
2688 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2689 sink_.WaitForEncodedFrame(ntp_time);
2690 ntp_time += kFrameIntervalMs;
2691
2692 video_stream_encoder_->TriggerQualityLow();
2693 video_source_.IncomingCapturedFrame(
2694 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2695 sink_.WaitForEncodedFrame(ntp_time);
2696 ntp_time += kFrameIntervalMs;
2697
2698 // Some resolution constraint should be set.
2699 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2700 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2701 kFrameWidth * kFrameHeight);
2702 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2703
2704 int pixel_count = video_source_.sink_wants().max_pixel_count;
2705 // Triggering a CPU underuse should not change the sink wants since it has
2706 // not been overused for resolution since we changed degradation preference.
2707 video_stream_encoder_->TriggerCpuUnderuse();
2708 video_source_.IncomingCapturedFrame(
2709 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2710 sink_.WaitForEncodedFrame(ntp_time);
2711 ntp_time += kFrameIntervalMs;
2712 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2713 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2714
Evan Shrubsole64469032020-06-11 10:45:29 +02002715 // Change the degradation preference back. CPU underuse should not adapt since
2716 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002717 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002718 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2719 video_source_.IncomingCapturedFrame(
2720 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2721 sink_.WaitForEncodedFrame(ntp_time);
2722 ntp_time += 100;
2723 // Resolution adaptations is gone after changing degradation preference.
2724 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2725 EXPECT_EQ(std::numeric_limits<int>::max(),
2726 video_source_.sink_wants().max_pixel_count);
2727 // The fps adaptation from above is now back.
2728 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2729
2730 // Trigger CPU underuse.
2731 video_stream_encoder_->TriggerCpuUnderuse();
2732 video_source_.IncomingCapturedFrame(
2733 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2734 sink_.WaitForEncodedFrame(ntp_time);
2735 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002736 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2737
2738 // Trigger QP underuse, fps should return to normal.
2739 video_stream_encoder_->TriggerQualityHigh();
2740 video_source_.IncomingCapturedFrame(
2741 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2742 sink_.WaitForEncodedFrame(ntp_time);
2743 ntp_time += kFrameIntervalMs;
2744 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002745
2746 video_stream_encoder_->Stop();
2747}
2748
mflodmancc3d4422017-08-03 08:27:51 -07002749TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002751 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002752 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002753
sprangc5d62e22017-04-02 23:53:04 -07002754 const int kFrameWidth = 1280;
2755 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002756
Åsa Persson8c1bf952018-09-13 10:42:19 +02002757 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002758
kthelgason5e13d412016-12-01 03:59:51 -08002759 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002760 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002761 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002762 frame_timestamp += kFrameIntervalMs;
2763
perkj803d97f2016-11-01 11:45:46 -07002764 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002765 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002766 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002767 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002768 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002769 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002770
asapersson0944a802017-04-07 00:57:58 -07002771 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002772 // wanted resolution.
2773 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2774 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2775 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002776 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002777
2778 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002779 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002780 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002781 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002782 // Give the encoder queue time to process the change in degradation preference
2783 // by waiting for an encoded frame.
2784 new_video_source.IncomingCapturedFrame(
2785 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2786 sink_.WaitForEncodedFrame(frame_timestamp);
2787 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002788 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002789 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002790
sprangc5d62e22017-04-02 23:53:04 -07002791 // Force an input frame rate to be available, or the adaptation call won't
2792 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002793 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002794 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002795 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002796 stats_proxy_->SetMockStats(stats);
2797
mflodmancc3d4422017-08-03 08:27:51 -07002798 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002799 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002800 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002801 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002802 frame_timestamp += kFrameIntervalMs;
2803
2804 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002805 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002806 EXPECT_EQ(std::numeric_limits<int>::max(),
2807 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002808 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002809
asapersson02465b82017-04-10 01:12:52 -07002810 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002811 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2812 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002813 // Give the encoder queue time to process the change in degradation preference
2814 // by waiting for an encoded frame.
2815 new_video_source.IncomingCapturedFrame(
2816 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2817 sink_.WaitForEncodedFrame(frame_timestamp);
2818 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002819 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002820
mflodmancc3d4422017-08-03 08:27:51 -07002821 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002822 new_video_source.IncomingCapturedFrame(
2823 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002824 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002825 frame_timestamp += kFrameIntervalMs;
2826
2827 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002828 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002829
2830 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002831 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002832 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002833 // Give the encoder queue time to process the change in degradation preference
2834 // by waiting for an encoded frame.
2835 new_video_source.IncomingCapturedFrame(
2836 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2837 sink_.WaitForEncodedFrame(frame_timestamp);
2838 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002839 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2840 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002841 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002842 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002843
2844 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002845 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002846 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002847 // Give the encoder queue time to process the change in degradation preference
2848 // by waiting for an encoded frame.
2849 new_video_source.IncomingCapturedFrame(
2850 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2851 sink_.WaitForEncodedFrame(frame_timestamp);
2852 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002853 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2854 EXPECT_EQ(std::numeric_limits<int>::max(),
2855 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002856 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002857
mflodmancc3d4422017-08-03 08:27:51 -07002858 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002859}
2860
mflodmancc3d4422017-08-03 08:27:51 -07002861TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002863 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002864
asaperssonfab67072017-04-04 05:51:49 -07002865 const int kWidth = 1280;
2866 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002867 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002868 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002869 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2870 EXPECT_FALSE(stats.bw_limited_resolution);
2871 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2872
2873 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002874 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002875 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002876 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002877
2878 stats = stats_proxy_->GetStats();
2879 EXPECT_TRUE(stats.bw_limited_resolution);
2880 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2881
2882 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002883 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002884 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002885 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002886
2887 stats = stats_proxy_->GetStats();
2888 EXPECT_FALSE(stats.bw_limited_resolution);
2889 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2890 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2891
mflodmancc3d4422017-08-03 08:27:51 -07002892 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002893}
2894
mflodmancc3d4422017-08-03 08:27:51 -07002895TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002896 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002897 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002898
2899 const int kWidth = 1280;
2900 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002901 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002902 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002903 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2904 EXPECT_FALSE(stats.cpu_limited_resolution);
2905 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2906
2907 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002908 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002909 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002910 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002911
2912 stats = stats_proxy_->GetStats();
2913 EXPECT_TRUE(stats.cpu_limited_resolution);
2914 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2915
2916 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002917 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002918 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002919 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002920
2921 stats = stats_proxy_->GetStats();
2922 EXPECT_FALSE(stats.cpu_limited_resolution);
2923 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002924 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002925
mflodmancc3d4422017-08-03 08:27:51 -07002926 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002927}
2928
mflodmancc3d4422017-08-03 08:27:51 -07002929TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002930 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002931 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002932
asaperssonfab67072017-04-04 05:51:49 -07002933 const int kWidth = 1280;
2934 const int kHeight = 720;
2935 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002936 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002937 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002938 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002939 EXPECT_FALSE(stats.cpu_limited_resolution);
2940 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2941
asaperssonfab67072017-04-04 05:51:49 -07002942 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002943 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002944 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002945 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002946 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002947 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002948 EXPECT_TRUE(stats.cpu_limited_resolution);
2949 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2950
2951 // Set new source with adaptation still enabled.
2952 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002953 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002954 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002955
asaperssonfab67072017-04-04 05:51:49 -07002956 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002957 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002958 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002959 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002960 EXPECT_TRUE(stats.cpu_limited_resolution);
2961 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2962
2963 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002964 video_stream_encoder_->SetSource(&new_video_source,
2965 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002966
asaperssonfab67072017-04-04 05:51:49 -07002967 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002968 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002969 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002970 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002971 EXPECT_FALSE(stats.cpu_limited_resolution);
2972 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2973
2974 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002975 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002976 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002977
asaperssonfab67072017-04-04 05:51:49 -07002978 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002979 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002980 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002981 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002982 EXPECT_TRUE(stats.cpu_limited_resolution);
2983 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2984
asaperssonfab67072017-04-04 05:51:49 -07002985 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002986 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002987 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002988 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002989 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002990 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002991 EXPECT_FALSE(stats.cpu_limited_resolution);
2992 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002993 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002994
mflodmancc3d4422017-08-03 08:27:51 -07002995 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002996}
2997
mflodmancc3d4422017-08-03 08:27:51 -07002998TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003000 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003001
asaperssonfab67072017-04-04 05:51:49 -07003002 const int kWidth = 1280;
3003 const int kHeight = 720;
3004 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003005 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003006 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003007 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003008 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003009 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003010
3011 // Set new source with adaptation still enabled.
3012 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003013 video_stream_encoder_->SetSource(&new_video_source,
3014 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003015
asaperssonfab67072017-04-04 05:51:49 -07003016 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003017 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003018 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003019 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003020 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003021 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003022
asaperssonfab67072017-04-04 05:51:49 -07003023 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003024 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003025 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003026 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003027 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003028 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003029 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003030 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003031
asaperssonfab67072017-04-04 05:51:49 -07003032 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003033 video_stream_encoder_->SetSource(&new_video_source,
3034 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003035
asaperssonfab67072017-04-04 05:51:49 -07003036 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003037 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003038 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003039 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003040 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003041 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003042
asapersson02465b82017-04-10 01:12:52 -07003043 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003044 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003045 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003046
asaperssonfab67072017-04-04 05:51:49 -07003047 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003048 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003049 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003050 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003051 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003052 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3053 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003054
mflodmancc3d4422017-08-03 08:27:51 -07003055 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003056}
3057
mflodmancc3d4422017-08-03 08:27:51 -07003058TEST_F(VideoStreamEncoderTest,
3059 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003061 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003062
3063 const int kWidth = 1280;
3064 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003065 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003066 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003067 video_source_.IncomingCapturedFrame(
3068 CreateFrame(timestamp_ms, kWidth, kHeight));
3069 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003070 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3071 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3072 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3073
3074 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003075 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003076 timestamp_ms += kFrameIntervalMs;
3077 video_source_.IncomingCapturedFrame(
3078 CreateFrame(timestamp_ms, kWidth, kHeight));
3079 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003080 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3081 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3082 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3083
3084 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003085 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003086 timestamp_ms += kFrameIntervalMs;
3087 video_source_.IncomingCapturedFrame(
3088 CreateFrame(timestamp_ms, kWidth, kHeight));
3089 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003090 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3091 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3092 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3093
Niels Möller4db138e2018-04-19 09:04:13 +02003094 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003095 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003096
3097 VideoEncoderConfig video_encoder_config;
3098 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3099 // Make format different, to force recreation of encoder.
3100 video_encoder_config.video_format.parameters["foo"] = "foo";
3101 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003102 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003103 timestamp_ms += kFrameIntervalMs;
3104 video_source_.IncomingCapturedFrame(
3105 CreateFrame(timestamp_ms, kWidth, kHeight));
3106 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003107 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3108 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3109 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3110
mflodmancc3d4422017-08-03 08:27:51 -07003111 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003112}
3113
mflodmancc3d4422017-08-03 08:27:51 -07003114TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003115 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003117 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003118
3119 const int kWidth = 1280;
3120 const int kHeight = 720;
3121 int sequence = 1;
3122
3123 // Enable BALANCED preference, no initial limitation.
3124 test::FrameForwarder source;
3125 video_stream_encoder_->SetSource(&source,
3126 webrtc::DegradationPreference::BALANCED);
3127 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3128 WaitForEncodedFrame(sequence++);
3129 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3130 EXPECT_FALSE(stats.cpu_limited_resolution);
3131 EXPECT_FALSE(stats.cpu_limited_framerate);
3132 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3133
3134 // Trigger CPU overuse, should now adapt down.
3135 video_stream_encoder_->TriggerCpuOveruse();
3136 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3137 WaitForEncodedFrame(sequence++);
3138 stats = stats_proxy_->GetStats();
3139 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3140
3141 // Set new degradation preference should clear restrictions since we changed
3142 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003143 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003144 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3145 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3146 WaitForEncodedFrame(sequence++);
3147 stats = stats_proxy_->GetStats();
3148 EXPECT_FALSE(stats.cpu_limited_resolution);
3149 EXPECT_FALSE(stats.cpu_limited_framerate);
3150 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3151
3152 // Force an input frame rate to be available, or the adaptation call won't
3153 // know what framerate to adapt from.
3154 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3155 mock_stats.input_frame_rate = 30;
3156 stats_proxy_->SetMockStats(mock_stats);
3157 video_stream_encoder_->TriggerCpuOveruse();
3158 stats_proxy_->ResetMockStats();
3159 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3160 WaitForEncodedFrame(sequence++);
3161
3162 // We have now adapted once.
3163 stats = stats_proxy_->GetStats();
3164 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3165
3166 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003167 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3168 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003169 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3170 WaitForEncodedFrame(sequence++);
3171 stats = stats_proxy_->GetStats();
3172 EXPECT_FALSE(stats.cpu_limited_resolution);
3173 EXPECT_FALSE(stats.cpu_limited_framerate);
3174 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3175
3176 video_stream_encoder_->Stop();
3177}
3178
3179TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003180 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003181 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003182 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003183
asapersson0944a802017-04-07 00:57:58 -07003184 const int kWidth = 1280;
3185 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003186 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003187
asaperssonfab67072017-04-04 05:51:49 -07003188 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003189 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003190 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003191 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003192 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003193 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3194
asapersson02465b82017-04-10 01:12:52 -07003195 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003196 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003197 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003198 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003199 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003200 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003201 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003202 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3203
3204 // Set new source with adaptation still enabled.
3205 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003206 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003207 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003208
3209 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003210 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003211 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003212 stats = stats_proxy_->GetStats();
3213 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003214 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003215 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3216
sprangc5d62e22017-04-02 23:53:04 -07003217 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003218 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003219 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003220 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003221 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003222 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003223 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003224 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003225 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003226 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003227 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3228
sprangc5d62e22017-04-02 23:53:04 -07003229 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003230 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003231 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3232 mock_stats.input_frame_rate = 30;
3233 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003234 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003235 stats_proxy_->ResetMockStats();
3236
3237 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003238 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003239 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003240
3241 // Framerate now adapted.
3242 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003243 EXPECT_FALSE(stats.cpu_limited_resolution);
3244 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003245 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3246
3247 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003248 video_stream_encoder_->SetSource(&new_video_source,
3249 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003250 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003251 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003252 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003253
3254 stats = stats_proxy_->GetStats();
3255 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003256 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003257 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3258
3259 // Try to trigger overuse. Should not succeed.
3260 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003261 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003262 stats_proxy_->ResetMockStats();
3263
3264 stats = stats_proxy_->GetStats();
3265 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003266 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003267 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3268
3269 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003270 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003271 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003272 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003273 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003274 stats = stats_proxy_->GetStats();
3275 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003276 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003277 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003278
3279 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003280 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003281 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003282 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003283 stats = stats_proxy_->GetStats();
3284 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003285 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003286 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3287
3288 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003290 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003291 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003292 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003293 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003294 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003295 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003296 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003297 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003298 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3299
3300 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003301 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003302 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003303 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003304 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003305 stats = stats_proxy_->GetStats();
3306 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003307 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003308 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003309 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003310
mflodmancc3d4422017-08-03 08:27:51 -07003311 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003312}
3313
mflodmancc3d4422017-08-03 08:27:51 -07003314TEST_F(VideoStreamEncoderTest,
3315 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003316 const int kWidth = 1280;
3317 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003319 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003320
asaperssonfab67072017-04-04 05:51:49 -07003321 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003322 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003323
asaperssonfab67072017-04-04 05:51:49 -07003324 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003325 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003326
asaperssonfab67072017-04-04 05:51:49 -07003327 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003328 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003329
asaperssonfab67072017-04-04 05:51:49 -07003330 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003331 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003332
kthelgason876222f2016-11-29 01:44:11 -08003333 // Expect a scale down.
3334 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003335 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003336
asapersson02465b82017-04-10 01:12:52 -07003337 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003338 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003339 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003340 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003341
asaperssonfab67072017-04-04 05:51:49 -07003342 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003343 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003344 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003345 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003346
asaperssonfab67072017-04-04 05:51:49 -07003347 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003348 EXPECT_EQ(std::numeric_limits<int>::max(),
3349 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003350
asaperssonfab67072017-04-04 05:51:49 -07003351 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003352 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003353 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003354 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003355
asapersson02465b82017-04-10 01:12:52 -07003356 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003357 EXPECT_EQ(std::numeric_limits<int>::max(),
3358 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003359
mflodmancc3d4422017-08-03 08:27:51 -07003360 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003361}
3362
mflodmancc3d4422017-08-03 08:27:51 -07003363TEST_F(VideoStreamEncoderTest,
3364 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003365 const int kWidth = 1280;
3366 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003367 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003368 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003369
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003370 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003371 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003373 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003374
3375 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003376 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003377 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003378 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3379 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3380
3381 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003382 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003383 EXPECT_THAT(source.sink_wants(),
3384 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003385 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3386 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3388
3389 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003390 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003391 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3392 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3393 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3394
mflodmancc3d4422017-08-03 08:27:51 -07003395 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003396}
3397
mflodmancc3d4422017-08-03 08:27:51 -07003398TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003399 const int kWidth = 1280;
3400 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003402 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003403
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003404 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003405 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003406 video_stream_encoder_->SetSource(&source,
3407 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003408 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3409 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003410 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003411
3412 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003413 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003414 EXPECT_THAT(source.sink_wants(),
3415 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003416 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3417 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3418 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3419
3420 // Trigger adapt down for same input resolution, expect no change.
3421 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3422 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003423 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003424 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3426 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3427
3428 // Trigger adapt down for larger input resolution, expect no change.
3429 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3430 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003432 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
mflodmancc3d4422017-08-03 08:27:51 -07003436 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003437}
3438
mflodmancc3d4422017-08-03 08:27:51 -07003439TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003440 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3441 const int kWidth = 640;
3442 const int kHeight = 360;
3443 const int64_t kFrameIntervalMs = 150;
3444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003445 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003446
3447 // Enable BALANCED preference, no initial limitation.
3448 AdaptingFrameForwarder source(&time_controller_);
3449 source.set_adaptation_enabled(true);
3450 video_stream_encoder_->SetSource(&source,
3451 webrtc::DegradationPreference::BALANCED);
3452
3453 int64_t timestamp_ms = kFrameIntervalMs;
3454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3455 sink_.WaitForEncodedFrame(kWidth, kHeight);
3456 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Trigger adapt down, expect reduced fps (640x360@15fps).
3462 video_stream_encoder_->TriggerQualityLow();
3463 timestamp_ms += kFrameIntervalMs;
3464 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3465 sink_.WaitForEncodedFrame(timestamp_ms);
3466 EXPECT_THAT(source.sink_wants(),
3467 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3470 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3471
3472 // Source requests 270p, expect reduced resolution (480x270@15fps).
3473 source.OnOutputFormatRequest(480, 270);
3474 timestamp_ms += kFrameIntervalMs;
3475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3476 WaitForEncodedFrame(480, 270);
3477 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3478
3479 // Trigger adapt down, expect reduced fps (480x270@10fps).
3480 video_stream_encoder_->TriggerQualityLow();
3481 timestamp_ms += kFrameIntervalMs;
3482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3483 sink_.WaitForEncodedFrame(timestamp_ms);
3484 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3486 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3487 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3488
3489 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3490 source.OnOutputFormatRequest(320, 180);
3491 timestamp_ms += kFrameIntervalMs;
3492 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3493 WaitForEncodedFrame(320, 180);
3494 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3495
3496 // Trigger adapt down, expect reduced fps (320x180@7fps).
3497 video_stream_encoder_->TriggerQualityLow();
3498 timestamp_ms += kFrameIntervalMs;
3499 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3500 sink_.WaitForEncodedFrame(timestamp_ms);
3501 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3503 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3504 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3505
3506 // Source requests VGA, expect increased resolution (640x360@7fps).
3507 source.OnOutputFormatRequest(640, 360);
3508 timestamp_ms += kFrameIntervalMs;
3509 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3510 WaitForEncodedFrame(timestamp_ms);
3511 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3512
3513 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3514 video_stream_encoder_->TriggerQualityHigh();
3515 timestamp_ms += kFrameIntervalMs;
3516 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3517 WaitForEncodedFrame(timestamp_ms);
3518 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3521 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3522
3523 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3524 video_stream_encoder_->TriggerQualityHigh();
3525 timestamp_ms += kFrameIntervalMs;
3526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3527 WaitForEncodedFrame(timestamp_ms);
3528 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3530 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3531 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3532
3533 // Trigger adapt up, expect increased fps (640x360@maxfps).
3534 video_stream_encoder_->TriggerQualityHigh();
3535 timestamp_ms += kFrameIntervalMs;
3536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3537 WaitForEncodedFrame(timestamp_ms);
3538 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3539 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3541 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3542
3543 video_stream_encoder_->Stop();
3544}
3545
3546TEST_F(VideoStreamEncoderTest,
3547 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3548 const int kWidth = 1280;
3549 const int kHeight = 720;
3550 const int64_t kFrameIntervalMs = 150;
3551 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003552 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003553
3554 // Enable BALANCED preference, no initial limitation.
3555 AdaptingFrameForwarder source(&time_controller_);
3556 source.set_adaptation_enabled(true);
3557 video_stream_encoder_->SetSource(&source,
3558 webrtc::DegradationPreference::BALANCED);
3559
3560 int64_t timestamp_ms = kFrameIntervalMs;
3561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3562 sink_.WaitForEncodedFrame(kWidth, kHeight);
3563 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3564 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3566 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3567
3568 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3569 video_stream_encoder_->TriggerQualityLow();
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 sink_.WaitForEncodedFrame(timestamp_ms);
3573 EXPECT_THAT(source.sink_wants(),
3574 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3576 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3577 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3578
3579 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3580 video_stream_encoder_->TriggerQualityLow();
3581 timestamp_ms += kFrameIntervalMs;
3582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3583 sink_.WaitForEncodedFrame(timestamp_ms);
3584 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3586 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3587 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3588
3589 // Trigger adapt down, expect reduced fps (640x360@15fps).
3590 video_stream_encoder_->TriggerQualityLow();
3591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
3594 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3597 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3598
3599 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3600 source.OnOutputFormatRequest(320, 180);
3601 timestamp_ms += kFrameIntervalMs;
3602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3603 WaitForEncodedFrame(320, 180);
3604 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3605 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3606
3607 // Trigger adapt down, expect reduced fps (320x180@7fps).
3608 video_stream_encoder_->TriggerCpuOveruse();
3609 timestamp_ms += kFrameIntervalMs;
3610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3611 WaitForEncodedFrame(timestamp_ms);
3612 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3615 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3617 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3618 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3619
3620 // Source requests HD, expect increased resolution (640x360@7fps).
3621 source.OnOutputFormatRequest(1280, 720);
3622 timestamp_ms += kFrameIntervalMs;
3623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3624 WaitForEncodedFrame(timestamp_ms);
3625 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3626 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3627
3628 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3629 video_stream_encoder_->TriggerCpuUnderuse();
3630 timestamp_ms += kFrameIntervalMs;
3631 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3632 WaitForEncodedFrame(timestamp_ms);
3633 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3636 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3637 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3638 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3639 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3640
3641 // Trigger adapt up, expect increased fps (640x360@maxfps).
3642 video_stream_encoder_->TriggerQualityHigh();
3643 video_stream_encoder_->TriggerCpuUnderuse();
3644 timestamp_ms += kFrameIntervalMs;
3645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3646 WaitForEncodedFrame(timestamp_ms);
3647 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3649 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3650 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3651 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3652 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3653 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3654
3655 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3656 video_stream_encoder_->TriggerQualityHigh();
3657 video_stream_encoder_->TriggerCpuUnderuse();
3658 timestamp_ms += kFrameIntervalMs;
3659 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3660 WaitForEncodedFrame(timestamp_ms);
3661 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3662 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3664 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3666 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3667 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3668
3669 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3670 video_stream_encoder_->TriggerQualityHigh();
3671 video_stream_encoder_->TriggerCpuUnderuse();
3672 timestamp_ms += kFrameIntervalMs;
3673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3674 WaitForEncodedFrame(timestamp_ms);
3675 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3679 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3680 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3681 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3682
3683 video_stream_encoder_->Stop();
3684}
3685
3686TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003687 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003688 const int kWidth = 1280;
3689 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003691 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003692
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003693 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003694 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003695 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003696 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003697
3698 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003699 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003700 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3703
3704 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003705 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003706 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003707 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3708 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3709
mflodmancc3d4422017-08-03 08:27:51 -07003710 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003711}
3712
mflodmancc3d4422017-08-03 08:27:51 -07003713TEST_F(VideoStreamEncoderTest,
3714 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003715 const int kWidth = 1280;
3716 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003718 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003719
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003720 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003721 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003722 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003723 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003724
3725 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003726 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003727 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3730
3731 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003732 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003733 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003734 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003735 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3736
mflodmancc3d4422017-08-03 08:27:51 -07003737 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003738}
3739
mflodmancc3d4422017-08-03 08:27:51 -07003740TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003741 const int kWidth = 1280;
3742 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003745
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003746 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003747 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003748 video_stream_encoder_->SetSource(&source,
3749 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003750
3751 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3752 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003753 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003754 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3755 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3756 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3757
3758 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003759 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003761 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3762 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3763 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3764
mflodmancc3d4422017-08-03 08:27:51 -07003765 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003766}
3767
mflodmancc3d4422017-08-03 08:27:51 -07003768TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003769 const int kWidth = 1280;
3770 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003773
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003774 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003775 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003776 video_stream_encoder_->SetSource(&source,
3777 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003778
3779 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3780 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003781 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3783 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3784 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3785
3786 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003787 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003788 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003789 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3790 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3791 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3792
mflodmancc3d4422017-08-03 08:27:51 -07003793 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003794}
3795
mflodmancc3d4422017-08-03 08:27:51 -07003796TEST_F(VideoStreamEncoderTest,
3797 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003798 const int kWidth = 1280;
3799 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003802
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003803 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003804 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003805 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003806 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003807 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003808
3809 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003810 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003811 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003812 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3813 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3814
3815 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003816 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003817 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003818 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003819 EXPECT_THAT(source.sink_wants(),
3820 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003821 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3822 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823
3824 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003825 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003826 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003827 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3828 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3829 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3830
mflodmancc3d4422017-08-03 08:27:51 -07003831 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003832}
3833
mflodmancc3d4422017-08-03 08:27:51 -07003834TEST_F(VideoStreamEncoderTest,
3835 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003836 const int kWidth = 1280;
3837 const int kHeight = 720;
3838 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003840 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003841
3842 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3843 stats.input_frame_rate = kInputFps;
3844 stats_proxy_->SetMockStats(stats);
3845
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003846 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003847 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3848 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003849 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003850
3851 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003852 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003853 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3854 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003855 EXPECT_THAT(video_source_.sink_wants(),
3856 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003857
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003858 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003859 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003860 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003861 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003862 // Give the encoder queue time to process the change in degradation preference
3863 // by waiting for an encoded frame.
3864 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3865 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003866 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003867
3868 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003869 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003870 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3871 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003872 EXPECT_THAT(new_video_source.sink_wants(),
3873 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003874
3875 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003876 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003877 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003878
mflodmancc3d4422017-08-03 08:27:51 -07003879 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003880}
3881
mflodmancc3d4422017-08-03 08:27:51 -07003882TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003883 const int kWidth = 1280;
3884 const int kHeight = 720;
3885 const size_t kNumFrames = 10;
3886
Henrik Boström381d1092020-05-12 18:49:07 +02003887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003888 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003889
asaperssond0de2952017-04-21 01:47:31 -07003890 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003891 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003892 video_source_.set_adaptation_enabled(true);
3893
3894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3895 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3896
3897 int downscales = 0;
3898 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003899 video_source_.IncomingCapturedFrame(
3900 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3901 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003902
asaperssonfab67072017-04-04 05:51:49 -07003903 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003904 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003905 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003906 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003907
3908 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3909 ++downscales;
3910
3911 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3912 EXPECT_EQ(downscales,
3913 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3914 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003915 }
mflodmancc3d4422017-08-03 08:27:51 -07003916 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003917}
3918
mflodmancc3d4422017-08-03 08:27:51 -07003919TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003920 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3921 const int kWidth = 1280;
3922 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003924 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003925
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003926 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003927 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003928 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003929 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003930 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003931
Åsa Persson8c1bf952018-09-13 10:42:19 +02003932 int64_t timestamp_ms = kFrameIntervalMs;
3933 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003934 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003935 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003936 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3937 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3938
3939 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003940 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003941 timestamp_ms += kFrameIntervalMs;
3942 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3943 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003944 EXPECT_THAT(source.sink_wants(),
3945 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003946 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3947 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3948
3949 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003950 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003951 timestamp_ms += kFrameIntervalMs;
3952 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003953 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003954 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003955 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3956 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3957
3958 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003959 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003960 timestamp_ms += kFrameIntervalMs;
3961 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3962 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003963 EXPECT_THAT(source.sink_wants(),
3964 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003965 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3966 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3967
3968 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003969 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003970 timestamp_ms += kFrameIntervalMs;
3971 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003972 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003973 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003974 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3975 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3976
mflodmancc3d4422017-08-03 08:27:51 -07003977 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003978}
3979
mflodmancc3d4422017-08-03 08:27:51 -07003980TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003981 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3982 const int kWidth = 1280;
3983 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003985 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003986
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003987 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003988 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003989 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003990 video_stream_encoder_->SetSource(&source,
3991 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003992
Åsa Persson8c1bf952018-09-13 10:42:19 +02003993 int64_t timestamp_ms = kFrameIntervalMs;
3994 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003995 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003996 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003997 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3998 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3999
4000 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004001 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004002 timestamp_ms += kFrameIntervalMs;
4003 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4004 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004005 EXPECT_THAT(source.sink_wants(),
4006 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004007 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4008 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4009
4010 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004011 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004012 timestamp_ms += kFrameIntervalMs;
4013 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004014 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004015 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004016 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4017 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4018
4019 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004020 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004021 timestamp_ms += kFrameIntervalMs;
4022 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4023 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004024 EXPECT_THAT(source.sink_wants(),
4025 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004026 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4027 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4028
4029 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004030 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004031 timestamp_ms += kFrameIntervalMs;
4032 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004033 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004034 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4036 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4037
mflodmancc3d4422017-08-03 08:27:51 -07004038 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004039}
4040
Sergey Silkin41c650b2019-10-14 13:12:19 +02004041TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4042 fake_encoder_.SetResolutionBitrateLimits(
4043 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4044
Henrik Boström381d1092020-05-12 18:49:07 +02004045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004046 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4047 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4048 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4049 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050
4051 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004052 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004053 source.set_adaptation_enabled(true);
4054 video_stream_encoder_->SetSource(
4055 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4056
4057 // Insert 720p frame.
4058 int64_t timestamp_ms = kFrameIntervalMs;
4059 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4060 WaitForEncodedFrame(1280, 720);
4061
4062 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004064 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4065 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4066 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4067 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004068 video_stream_encoder_->TriggerQualityLow();
4069
4070 // Insert 720p frame. It should be downscaled and encoded.
4071 timestamp_ms += kFrameIntervalMs;
4072 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4073 WaitForEncodedFrame(960, 540);
4074
4075 // Trigger adapt up. Higher resolution should not be requested duo to lack
4076 // of bitrate.
4077 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004078 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004079
4080 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004081 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004082 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4083 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4084 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4085 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004086
4087 // Trigger adapt up. Higher resolution should be requested.
4088 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004089 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004090
4091 video_stream_encoder_->Stop();
4092}
4093
4094TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4095 fake_encoder_.SetResolutionBitrateLimits(
4096 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4097
4098 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004100 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4101 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4102 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4103 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004104
4105 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004106 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004107 source.set_adaptation_enabled(true);
4108 video_stream_encoder_->SetSource(
4109 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4110
4111 // Insert 720p frame. It should be dropped and lower resolution should be
4112 // requested.
4113 int64_t timestamp_ms = kFrameIntervalMs;
4114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4115 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004116 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004117
4118 // Insert 720p frame. It should be downscaled and encoded.
4119 timestamp_ms += kFrameIntervalMs;
4120 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4121 WaitForEncodedFrame(960, 540);
4122
4123 video_stream_encoder_->Stop();
4124}
4125
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004126class BalancedDegradationTest : public VideoStreamEncoderTest {
4127 protected:
4128 void SetupTest() {
4129 // Reset encoder for field trials to take effect.
4130 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004131 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004132
4133 // Enable BALANCED preference.
4134 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004135 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4136 }
4137
Asa Persson606d3cb2021-10-04 10:07:11 +02004138 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004139 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004140 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141 }
4142
Åsa Persson45b176f2019-09-30 11:19:05 +02004143 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004144 timestamp_ms_ += kFrameIntervalMs;
4145 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004146 }
4147
4148 void InsertFrameAndWaitForEncoded() {
4149 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004150 sink_.WaitForEncodedFrame(timestamp_ms_);
4151 }
4152
4153 const int kWidth = 640; // pixels:640x360=230400
4154 const int kHeight = 360;
4155 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4156 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004157 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004158};
4159
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004160TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004161 test::ScopedKeyValueConfig field_trials(
4162 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004163 "WebRTC-Video-BalancedDegradationSettings/"
4164 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4165 SetupTest();
4166
4167 // Force input frame rate.
4168 const int kInputFps = 24;
4169 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4170 stats.input_frame_rate = kInputFps;
4171 stats_proxy_->SetMockStats(stats);
4172
Åsa Persson45b176f2019-09-30 11:19:05 +02004173 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004174 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004175
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004176 // Trigger adapt down, expect scaled down framerate and resolution,
4177 // since Fps diff (input-requested:0) < threshold.
4178 video_stream_encoder_->TriggerQualityLow();
4179 EXPECT_THAT(source_.sink_wants(),
4180 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004181
4182 video_stream_encoder_->Stop();
4183}
4184
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004185TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004186 test::ScopedKeyValueConfig field_trials(
4187 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004188 "WebRTC-Video-BalancedDegradationSettings/"
4189 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4190 SetupTest();
4191
4192 // Force input frame rate.
4193 const int kInputFps = 25;
4194 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4195 stats.input_frame_rate = kInputFps;
4196 stats_proxy_->SetMockStats(stats);
4197
Åsa Persson45b176f2019-09-30 11:19:05 +02004198 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004199 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004200
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004201 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4202 // Fps diff (input-requested:1) == threshold.
4203 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004204 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004205
4206 video_stream_encoder_->Stop();
4207}
4208
4209TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004210 test::ScopedKeyValueConfig field_trials(
4211 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004212 "WebRTC-Video-BalancedDegradationSettings/"
4213 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4214 SetupTest();
4215
4216 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4217
Åsa Persson45b176f2019-09-30 11:19:05 +02004218 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004219 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004220
4221 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4222 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004223 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004224
4225 video_stream_encoder_->Stop();
4226}
4227
Åsa Perssonccfb3402019-09-25 15:13:04 +02004228TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004229 test::ScopedKeyValueConfig field_trials(
4230 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004231 "WebRTC-Video-BalancedDegradationSettings/"
4232 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004233 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004234
Asa Persson606d3cb2021-10-04 10:07:11 +02004235 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4236 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4237 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004238
Åsa Persson45b176f2019-09-30 11:19:05 +02004239 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004240 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004241 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4242
4243 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4244 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004245 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004246 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004247 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4248
4249 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4250 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004251 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004252 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004253 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4254
Åsa Persson30ab0152019-08-27 12:22:33 +02004255 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4256 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004257 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004258 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004259 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004260 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4261
4262 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004263 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004264 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004265 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004266
Åsa Persson30ab0152019-08-27 12:22:33 +02004267 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004268 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004269 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004270 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004271 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004272 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4273
4274 video_stream_encoder_->Stop();
4275}
4276
Åsa Perssonccfb3402019-09-25 15:13:04 +02004277TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004279 test::ScopedKeyValueConfig field_trials(
4280 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004281 "WebRTC-Video-BalancedDegradationSettings/"
4282 "pixels:57600|129600|230400,fps:7|24|24/");
4283 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004284 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004285
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004286 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004287
4288 // Insert frame, expect scaled down:
4289 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4290 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004291 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004292 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4293 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4294
4295 // Insert frame, expect scaled down:
4296 // resolution (320x180@24fps).
4297 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004298 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004299 EXPECT_LT(source_.sink_wants().max_pixel_count,
4300 source_.last_wants().max_pixel_count);
4301 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4302
4303 // Frame should not be dropped (min pixels per frame reached).
4304 InsertFrameAndWaitForEncoded();
4305
4306 video_stream_encoder_->Stop();
4307}
4308
4309TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004310 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004311 test::ScopedKeyValueConfig field_trials(
4312 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004313 "WebRTC-Video-BalancedDegradationSettings/"
4314 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004315 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004316
Asa Persson606d3cb2021-10-04 10:07:11 +02004317 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4318 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4319 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004320
Åsa Persson45b176f2019-09-30 11:19:05 +02004321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004322 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004323 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4324
4325 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4326 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004327 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004328 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004329 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4330
4331 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4332 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004333 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004334 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004335 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4338 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004339 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004340 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004341 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4342
Åsa Persson30ab0152019-08-27 12:22:33 +02004343 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4344 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004345 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004346 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004347 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4348
4349 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4350 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004351 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004352 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4353
4354 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004355 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004356 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004357 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004358 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004359 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4360
4361 video_stream_encoder_->Stop();
4362}
4363
Åsa Perssonccfb3402019-09-25 15:13:04 +02004364TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004365 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004366 test::ScopedKeyValueConfig field_trials(
4367 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004368 "WebRTC-Video-BalancedDegradationSettings/"
4369 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004370 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004371
Asa Persson606d3cb2021-10-04 10:07:11 +02004372 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4373 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4374 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4375 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4376 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004377
Åsa Persson45b176f2019-09-30 11:19:05 +02004378 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004379 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004380 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4381
4382 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4383 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004384 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004385 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004386 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4387
4388 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4389 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004390 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004391 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004392 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4393
4394 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4395 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004396 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004397 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004398 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4399
4400 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4401 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004402 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004403 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4404
4405 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004406 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004407 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004408 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004409 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004410 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
4412 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004413 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004414 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004415 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004416 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4417
4418 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004419 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004420 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004421 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004422 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004423 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4424
Åsa Persson1b247f12019-08-14 17:26:39 +02004425 video_stream_encoder_->Stop();
4426}
4427
mflodmancc3d4422017-08-03 08:27:51 -07004428TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004429 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4430 const int kWidth = 1280;
4431 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004433 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004434
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004435 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004436 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004437 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004438 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004439 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004440
Åsa Persson8c1bf952018-09-13 10:42:19 +02004441 int64_t timestamp_ms = kFrameIntervalMs;
4442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004443 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004444 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004445 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4447 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4449
4450 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004451 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004452 timestamp_ms += kFrameIntervalMs;
4453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4454 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004455 EXPECT_THAT(source.sink_wants(),
4456 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
4462 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004463 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4470 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4471 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4472
Jonathan Yubc771b72017-12-08 17:04:29 -08004473 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004474 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004475 timestamp_ms += kFrameIntervalMs;
4476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4477 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004479 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004482 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4483
Jonathan Yubc771b72017-12-08 17:04:29 -08004484 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004485 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004486 timestamp_ms += kFrameIntervalMs;
4487 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4488 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004489 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004490 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004491 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4492 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4493 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4494 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4495
Jonathan Yubc771b72017-12-08 17:04:29 -08004496 // Trigger quality adapt down, expect no change (min resolution reached).
4497 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004498 timestamp_ms += kFrameIntervalMs;
4499 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4500 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004501 EXPECT_THAT(source.sink_wants(), FpsMax());
4502 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004503 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4505 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4506 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4507
Evan Shrubsole64469032020-06-11 10:45:29 +02004508 // Trigger quality adapt up, expect upscaled resolution (480x270).
4509 video_stream_encoder_->TriggerQualityHigh();
4510 timestamp_ms += kFrameIntervalMs;
4511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4512 WaitForEncodedFrame(timestamp_ms);
4513 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4514 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4516 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4517 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4518
4519 // Trigger quality and cpu adapt up since both are most limited, expect
4520 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004521 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004522 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004523 timestamp_ms += kFrameIntervalMs;
4524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4525 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004526 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004527 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4529 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004530 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004531
Evan Shrubsole64469032020-06-11 10:45:29 +02004532 // Trigger quality and cpu adapt up since both are most limited, expect
4533 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004534 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004535 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004536 timestamp_ms += kFrameIntervalMs;
4537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4538 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004539 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004540 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004541 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004543 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4544 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004545
Evan Shrubsole64469032020-06-11 10:45:29 +02004546 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4547 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004548 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004549 timestamp_ms += kFrameIntervalMs;
4550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004552 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004553 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004555 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004556 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004557
4558 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004559 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004560 timestamp_ms += kFrameIntervalMs;
4561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004562 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004563 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004565 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004567 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004568 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004569
mflodmancc3d4422017-08-03 08:27:51 -07004570 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004571}
4572
mflodmancc3d4422017-08-03 08:27:51 -07004573TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004574 const int kWidth = 640;
4575 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004576
Henrik Boström381d1092020-05-12 18:49:07 +02004577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004578 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004579
perkj803d97f2016-11-01 11:45:46 -07004580 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004581 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004582 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004583 }
4584
mflodmancc3d4422017-08-03 08:27:51 -07004585 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004586 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004587 video_source_.IncomingCapturedFrame(CreateFrame(
4588 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004589 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004590 }
4591
mflodmancc3d4422017-08-03 08:27:51 -07004592 video_stream_encoder_->Stop();
4593 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004594 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004595
Ying Wangef3998f2019-12-09 13:06:53 +01004596 EXPECT_METRIC_EQ(
4597 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4598 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004599 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4600}
4601
mflodmancc3d4422017-08-03 08:27:51 -07004602TEST_F(VideoStreamEncoderTest,
4603 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004605 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004606 const int kWidth = 640;
4607 const int kHeight = 360;
4608
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004609 video_stream_encoder_->SetSource(&video_source_,
4610 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004611
4612 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4613 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004614 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004615 }
4616
mflodmancc3d4422017-08-03 08:27:51 -07004617 video_stream_encoder_->Stop();
4618 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004619 stats_proxy_.reset();
4620
4621 EXPECT_EQ(0,
4622 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4623}
4624
Per Kjellanderdcef6412020-10-07 15:09:05 +02004625TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4626 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004627 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004628 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004629
4630 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004631 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004632 SimulcastRateAllocator(fake_encoder_.config())
4633 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004634 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004635
Henrik Boström381d1092020-05-12 18:49:07 +02004636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004637 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004638
sprang57c2fff2017-01-16 06:24:02 -08004639 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004640 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4641 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004642 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4643 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4644
Erik Språngd7329ca2019-02-21 21:19:53 +01004645 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004646 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004647 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004648
Per Kjellanderdcef6412020-10-07 15:09:05 +02004649 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004650 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004651 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4652 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004654 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004655
Per Kjellanderdcef6412020-10-07 15:09:05 +02004656 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004657 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004658 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004659 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004660 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4661 WaitForEncodedFrame(CurrentTimeMs());
4662 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004663 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004664 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004665
mflodmancc3d4422017-08-03 08:27:51 -07004666 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004667}
4668
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004669TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004670 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004671 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004672 kVideoLayersAllocation);
4673
4674 const int kDefaultFps = 30;
4675
4676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004677 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004678
4679 video_source_.IncomingCapturedFrame(
4680 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4681 WaitForEncodedFrame(CurrentTimeMs());
4682 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4683 VideoLayersAllocation last_layer_allocation =
4684 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004685 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004686 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4687
4688 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004689 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004690 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004691 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004692 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4693
Erik Språng9d69cbe2020-10-22 17:44:42 +02004694 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004695 int number_of_layers_allocation = 1;
4696 const int64_t start_time_ms = CurrentTimeMs();
4697 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4698 video_source_.IncomingCapturedFrame(
4699 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4700 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004701 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4702 number_of_layers_allocation = sink_.number_of_layers_allocations();
4703 VideoLayersAllocation new_allocation =
4704 sink_.GetLastVideoLayersAllocation();
4705 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4706 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4707 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4708 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4709 .target_bitrate_per_temporal_layer,
4710 last_layer_allocation.active_spatial_layers[0]
4711 .target_bitrate_per_temporal_layer);
4712 last_layer_allocation = new_allocation;
4713 }
4714 }
4715 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4716 video_stream_encoder_->Stop();
4717}
4718
4719TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004720 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4722 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4723 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004724 VideoEncoderConfig video_encoder_config;
4725 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4726 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004727 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004728 video_encoder_config.content_type =
4729 VideoEncoderConfig::ContentType::kRealtimeVideo;
4730 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004731 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004732 VideoEncoder::GetDefaultVp8Settings());
4733 for (auto& layer : video_encoder_config.simulcast_layers) {
4734 layer.num_temporal_layers = 2;
4735 }
4736 // Simulcast layers are used for enabling/disabling streams.
4737 video_encoder_config.simulcast_layers[0].active = true;
4738 video_encoder_config.simulcast_layers[1].active = false;
4739 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004740 ConfigureEncoder(std::move(video_encoder_config),
4741 VideoStreamEncoder::BitrateAllocationCallbackType::
4742 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004743
4744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004745 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004746
4747 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4748 WaitForEncodedFrame(CurrentTimeMs());
4749 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4750 VideoLayersAllocation last_layer_allocation =
4751 sink_.GetLastVideoLayersAllocation();
4752
4753 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4754 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4755 .target_bitrate_per_temporal_layer,
4756 SizeIs(2));
4757 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4758 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4759 video_stream_encoder_->Stop();
4760}
4761
4762TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004763 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4765 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4766 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004767 VideoEncoderConfig video_encoder_config;
4768 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4769 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004770 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004771 video_encoder_config.content_type =
4772 VideoEncoderConfig::ContentType::kRealtimeVideo;
4773 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004774 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775 VideoEncoder::GetDefaultVp8Settings());
4776 for (auto& layer : video_encoder_config.simulcast_layers) {
4777 layer.num_temporal_layers = 2;
4778 }
4779 // Simulcast layers are used for enabling/disabling streams.
4780 video_encoder_config.simulcast_layers[0].active = true;
4781 video_encoder_config.simulcast_layers[1].active = false;
4782 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004783 ConfigureEncoder(std::move(video_encoder_config),
4784 VideoStreamEncoder::BitrateAllocationCallbackType::
4785 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004786
4787 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004788 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004789
4790 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4791 WaitForEncodedFrame(CurrentTimeMs());
4792 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4793 VideoLayersAllocation last_layer_allocation =
4794 sink_.GetLastVideoLayersAllocation();
4795
4796 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4797 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4798 .target_bitrate_per_temporal_layer,
4799 SizeIs(2));
4800 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4801
4802 video_stream_encoder_->Stop();
4803}
4804
4805TEST_F(VideoStreamEncoderTest,
4806 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4807 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4808 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004809 VideoEncoderConfig video_encoder_config;
4810 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4811 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004812 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004813 video_encoder_config.content_type =
4814 VideoEncoderConfig::ContentType::kRealtimeVideo;
4815 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4816 vp9_settings.numberOfSpatialLayers = 2;
4817 vp9_settings.numberOfTemporalLayers = 2;
4818 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4819 vp9_settings.automaticResizeOn = false;
4820 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004821 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004822 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004823 ConfigureEncoder(std::move(video_encoder_config),
4824 VideoStreamEncoder::BitrateAllocationCallbackType::
4825 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004826
4827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004828 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004829
4830 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4831 WaitForEncodedFrame(CurrentTimeMs());
4832 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4833 VideoLayersAllocation last_layer_allocation =
4834 sink_.GetLastVideoLayersAllocation();
4835
4836 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4837 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4838 .target_bitrate_per_temporal_layer,
4839 SizeIs(2));
4840 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4842 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4843 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4844 .target_bitrate_per_temporal_layer,
4845 SizeIs(2));
4846 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4847 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4848 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4849
4850 // Since full SVC is used, expect the top layer to utilize the full target
4851 // rate.
4852 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4853 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004854 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004855 video_stream_encoder_->Stop();
4856}
4857
4858TEST_F(VideoStreamEncoderTest,
4859 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4860 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4861 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004862 VideoEncoderConfig video_encoder_config;
4863 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4864 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004865 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004866 video_encoder_config.content_type =
4867 VideoEncoderConfig::ContentType::kRealtimeVideo;
4868 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4869 vp9_settings.numberOfSpatialLayers = 2;
4870 vp9_settings.numberOfTemporalLayers = 2;
4871 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4872 vp9_settings.automaticResizeOn = false;
4873 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004874 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004875 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004876 ConfigureEncoder(std::move(video_encoder_config),
4877 VideoStreamEncoder::BitrateAllocationCallbackType::
4878 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004879
4880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004881 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004882
4883 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4884 WaitForEncodedFrame(CurrentTimeMs());
4885 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4886 VideoLayersAllocation last_layer_allocation =
4887 sink_.GetLastVideoLayersAllocation();
4888
4889 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4890 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4891 .target_bitrate_per_temporal_layer,
4892 SizeIs(1));
4893 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4894 .target_bitrate_per_temporal_layer,
4895 SizeIs(1));
4896 // Since full SVC is used, expect the top layer to utilize the full target
4897 // rate.
4898 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4899 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004900 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004901 video_stream_encoder_->Stop();
4902}
4903
4904TEST_F(VideoStreamEncoderTest,
4905 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4907 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004908 VideoEncoderConfig video_encoder_config;
4909 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4910 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004911 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004912 video_encoder_config.content_type =
4913 VideoEncoderConfig::ContentType::kRealtimeVideo;
4914 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4915 vp9_settings.numberOfSpatialLayers = 2;
4916 vp9_settings.numberOfTemporalLayers = 2;
4917 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4918 vp9_settings.automaticResizeOn = false;
4919 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004920 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004921 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004922 ConfigureEncoder(std::move(video_encoder_config),
4923 VideoStreamEncoder::BitrateAllocationCallbackType::
4924 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004925
4926 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004927 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004928
4929 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4930 WaitForEncodedFrame(CurrentTimeMs());
4931 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4932 VideoLayersAllocation last_layer_allocation =
4933 sink_.GetLastVideoLayersAllocation();
4934
4935 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4936 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4937 .target_bitrate_per_temporal_layer,
4938 SizeIs(2));
4939 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4940 .target_bitrate_per_temporal_layer,
4941 SizeIs(2));
4942 // Since KSVC is, spatial layers are independend except on key frames.
4943 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4944 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004945 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004946 video_stream_encoder_->Stop();
4947}
4948
4949TEST_F(VideoStreamEncoderTest,
4950 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4951 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4952 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4953 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004954 VideoEncoderConfig video_encoder_config;
4955 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4956 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004957 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004958 video_encoder_config.content_type =
4959 VideoEncoderConfig::ContentType::kRealtimeVideo;
4960 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4961 vp9_settings.numberOfSpatialLayers = 3;
4962 vp9_settings.numberOfTemporalLayers = 2;
4963 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4964 vp9_settings.automaticResizeOn = false;
4965 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004966 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004967 vp9_settings);
4968 // Simulcast layers are used for enabling/disabling streams.
4969 video_encoder_config.simulcast_layers.resize(3);
4970 video_encoder_config.simulcast_layers[0].active = false;
4971 video_encoder_config.simulcast_layers[1].active = true;
4972 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004973 ConfigureEncoder(std::move(video_encoder_config),
4974 VideoStreamEncoder::BitrateAllocationCallbackType::
4975 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004976
4977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004978 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004979
4980 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4981 WaitForEncodedFrame(CurrentTimeMs());
4982 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4983 VideoLayersAllocation last_layer_allocation =
4984 sink_.GetLastVideoLayersAllocation();
4985
4986 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4987 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4988 .target_bitrate_per_temporal_layer,
4989 SizeIs(2));
4990 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4991 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4992
4993 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4994 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4995 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4996 .target_bitrate_per_temporal_layer,
4997 SizeIs(2));
4998 // Since full SVC is used, expect the top layer to utilize the full target
4999 // rate.
5000 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
5001 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005002 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005003 video_stream_encoder_->Stop();
5004}
5005
5006TEST_F(VideoStreamEncoderTest,
5007 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5008 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5009 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5010 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005011 VideoEncoderConfig video_encoder_config;
5012 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5013 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005014 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005015 video_encoder_config.content_type =
5016 VideoEncoderConfig::ContentType::kRealtimeVideo;
5017 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5018 vp9_settings.numberOfSpatialLayers = 3;
5019 vp9_settings.numberOfTemporalLayers = 2;
5020 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5021 vp9_settings.automaticResizeOn = false;
5022 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005023 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005024 vp9_settings);
5025 // Simulcast layers are used for enabling/disabling streams.
5026 video_encoder_config.simulcast_layers.resize(3);
5027 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005028 ConfigureEncoder(std::move(video_encoder_config),
5029 VideoStreamEncoder::BitrateAllocationCallbackType::
5030 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005031
5032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005034
5035 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5036 WaitForEncodedFrame(CurrentTimeMs());
5037 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5038 VideoLayersAllocation last_layer_allocation =
5039 sink_.GetLastVideoLayersAllocation();
5040
5041 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5042 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5043 .target_bitrate_per_temporal_layer,
5044 SizeIs(2));
5045 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5046 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5047
5048 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5049 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5050 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5051 .target_bitrate_per_temporal_layer,
5052 SizeIs(2));
5053 video_stream_encoder_->Stop();
5054}
5055
5056TEST_F(VideoStreamEncoderTest,
5057 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5058 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5059 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5060 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005061 VideoEncoderConfig video_encoder_config;
5062 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5063 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005064 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005065 video_encoder_config.content_type =
5066 VideoEncoderConfig::ContentType::kRealtimeVideo;
5067 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5068 vp9_settings.numberOfSpatialLayers = 3;
5069 vp9_settings.numberOfTemporalLayers = 2;
5070 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5071 vp9_settings.automaticResizeOn = false;
5072 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005073 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005074 vp9_settings);
5075 // Simulcast layers are used for enabling/disabling streams.
5076 video_encoder_config.simulcast_layers.resize(3);
5077 video_encoder_config.simulcast_layers[0].active = false;
5078 video_encoder_config.simulcast_layers[1].active = false;
5079 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005080 ConfigureEncoder(std::move(video_encoder_config),
5081 VideoStreamEncoder::BitrateAllocationCallbackType::
5082 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005083
5084 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005085 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005086
5087 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5088 WaitForEncodedFrame(CurrentTimeMs());
5089 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5090 VideoLayersAllocation last_layer_allocation =
5091 sink_.GetLastVideoLayersAllocation();
5092
5093 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5094 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5095 .target_bitrate_per_temporal_layer,
5096 SizeIs(2));
5097 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5098 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5099 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5100 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005101 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005102 video_stream_encoder_->Stop();
5103}
5104
5105TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5106 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005107 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005108 kVideoLayersAllocation);
5109 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005110 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005111
5112 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5113 WaitForEncodedFrame(CurrentTimeMs());
5114 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5115 VideoLayersAllocation last_layer_allocation =
5116 sink_.GetLastVideoLayersAllocation();
5117
5118 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5119 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5120 .target_bitrate_per_temporal_layer,
5121 SizeIs(1));
5122 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5123 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005124 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005125 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5126 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5127 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5128 video_stream_encoder_->Stop();
5129}
5130
5131TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005132 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5133 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005134 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005135 kVideoLayersAllocation);
5136
5137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005138 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005139
5140 video_source_.IncomingCapturedFrame(
5141 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5142 WaitForEncodedFrame(CurrentTimeMs());
5143 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5144 VideoLayersAllocation last_layer_allocation =
5145 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005146 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005147 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5148 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5149 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005150 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005151
5152 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005153 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5154 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005155 video_source_.IncomingCapturedFrame(
5156 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5157 WaitForEncodedFrame(CurrentTimeMs());
5158
5159 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5160 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5161 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5162 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5163 .target_bitrate_per_temporal_layer[0],
5164 DataRate::Zero());
5165
5166 video_stream_encoder_->Stop();
5167}
5168
Per Kjellander4190ce92020-12-15 17:24:55 +01005169TEST_F(VideoStreamEncoderTest,
5170 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5171 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005172 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005173 kVideoLayersAllocation);
5174
5175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005176 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5177 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005178
5179 video_source_.IncomingCapturedFrame(
5180 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5181 WaitForEncodedFrame(CurrentTimeMs());
5182 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5183 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5184 SizeIs(2));
5185 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5186 codec_width_);
5187 EXPECT_EQ(
5188 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5189 codec_height_);
5190
5191 video_source_.IncomingCapturedFrame(
5192 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5193 WaitForEncodedFrame(CurrentTimeMs());
5194 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5195 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5196 SizeIs(2));
5197 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5198 codec_width_ / 2);
5199 EXPECT_EQ(
5200 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5201 codec_height_ / 2);
5202
5203 video_stream_encoder_->Stop();
5204}
5205
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005206TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5207 // 2 TLs configured, temporal layers supported by encoder.
5208 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005209 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005210 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005211 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005212 fake_encoder_.SetTemporalLayersSupported(0, true);
5213
5214 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005215 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005216 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005217 kNumTemporalLayers, /*temporal_id*/ 0,
5218 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005219 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005220 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005221 kNumTemporalLayers, /*temporal_id*/ 1,
5222 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005223 VideoBitrateAllocation expected_bitrate;
5224 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5225 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5226
5227 VerifyAllocatedBitrate(expected_bitrate);
5228 video_stream_encoder_->Stop();
5229}
5230
5231TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5232 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005233 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005234 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005235 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005236 fake_encoder_.SetTemporalLayersSupported(0, false);
5237
5238 // Temporal layers not supported by the encoder.
5239 // Total bitrate should be at ti:0.
5240 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005241 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005242
5243 VerifyAllocatedBitrate(expected_bitrate);
5244 video_stream_encoder_->Stop();
5245}
5246
5247TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005248 webrtc::test::ScopedKeyValueConfig field_trials(
5249 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005250 "WebRTC-Video-QualityScalerSettings/"
5251 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5252 // Reset encoder for field trials to take effect.
5253 ConfigureEncoder(video_encoder_config_.Copy());
5254
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005255 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005256 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005257 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005258 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005259 fake_encoder_.SetTemporalLayersSupported(0, true);
5260 fake_encoder_.SetTemporalLayersSupported(1, false);
5261
5262 const int kS0Bps = 150000;
5263 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005264 kS0Bps *
5265 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5266 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005267 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005268 kS0Bps *
5269 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5270 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005271 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005272 // Temporal layers not supported by si:1.
5273 VideoBitrateAllocation expected_bitrate;
5274 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5275 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5276 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5277
5278 VerifyAllocatedBitrate(expected_bitrate);
5279 video_stream_encoder_->Stop();
5280}
5281
Niels Möller7dc26b72017-12-06 10:27:48 +01005282TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5283 const int kFrameWidth = 1280;
5284 const int kFrameHeight = 720;
5285 const int kFramerate = 24;
5286
Henrik Boström381d1092020-05-12 18:49:07 +02005287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005288 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005289 test::FrameForwarder source;
5290 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005291 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005292
5293 // Insert a single frame, triggering initial configuration.
5294 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5295 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5296
5297 EXPECT_EQ(
5298 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5299 kDefaultFramerate);
5300
5301 // Trigger reconfigure encoder (without resetting the entire instance).
5302 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005303 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5304 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005305 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005306 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005307 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005308 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5309
5310 // Detector should be updated with fps limit from codec config.
5311 EXPECT_EQ(
5312 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5313 kFramerate);
5314
5315 // Trigger overuse, max framerate should be reduced.
5316 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5317 stats.input_frame_rate = kFramerate;
5318 stats_proxy_->SetMockStats(stats);
5319 video_stream_encoder_->TriggerCpuOveruse();
5320 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5321 int adapted_framerate =
5322 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5323 EXPECT_LT(adapted_framerate, kFramerate);
5324
5325 // Trigger underuse, max framerate should go back to codec configured fps.
5326 // Set extra low fps, to make sure it's actually reset, not just incremented.
5327 stats = stats_proxy_->GetStats();
5328 stats.input_frame_rate = adapted_framerate / 2;
5329 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005330 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005331 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5332 EXPECT_EQ(
5333 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5334 kFramerate);
5335
5336 video_stream_encoder_->Stop();
5337}
5338
5339TEST_F(VideoStreamEncoderTest,
5340 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5341 const int kFrameWidth = 1280;
5342 const int kFrameHeight = 720;
5343 const int kLowFramerate = 15;
5344 const int kHighFramerate = 25;
5345
Henrik Boström381d1092020-05-12 18:49:07 +02005346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005347 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005348 test::FrameForwarder source;
5349 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005350 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005351
5352 // Trigger initial configuration.
5353 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005354 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5355 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005356 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005357 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005358 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005359 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005360 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5361
5362 EXPECT_EQ(
5363 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5364 kLowFramerate);
5365
5366 // Trigger overuse, max framerate should be reduced.
5367 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5368 stats.input_frame_rate = kLowFramerate;
5369 stats_proxy_->SetMockStats(stats);
5370 video_stream_encoder_->TriggerCpuOveruse();
5371 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5372 int adapted_framerate =
5373 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5374 EXPECT_LT(adapted_framerate, kLowFramerate);
5375
5376 // Reconfigure the encoder with a new (higher max framerate), max fps should
5377 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005378 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005379 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5380 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005381 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005382 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5383
5384 EXPECT_EQ(
5385 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5386 adapted_framerate);
5387
5388 // Trigger underuse, max framerate should go back to codec configured fps.
5389 stats = stats_proxy_->GetStats();
5390 stats.input_frame_rate = adapted_framerate;
5391 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005392 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005393 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5394 EXPECT_EQ(
5395 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5396 kHighFramerate);
5397
5398 video_stream_encoder_->Stop();
5399}
5400
mflodmancc3d4422017-08-03 08:27:51 -07005401TEST_F(VideoStreamEncoderTest,
5402 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005403 const int kFrameWidth = 1280;
5404 const int kFrameHeight = 720;
5405 const int kFramerate = 24;
5406
Henrik Boström381d1092020-05-12 18:49:07 +02005407 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005408 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005409 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005410 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005411 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005412
5413 // Trigger initial configuration.
5414 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005415 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5416 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005417 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005418 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005419 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005420 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005421 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005422
Niels Möller7dc26b72017-12-06 10:27:48 +01005423 EXPECT_EQ(
5424 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5425 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005426
5427 // Trigger overuse, max framerate should be reduced.
5428 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5429 stats.input_frame_rate = kFramerate;
5430 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005431 video_stream_encoder_->TriggerCpuOveruse();
5432 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005433 int adapted_framerate =
5434 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005435 EXPECT_LT(adapted_framerate, kFramerate);
5436
5437 // Change degradation preference to not enable framerate scaling. Target
5438 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005439 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005440 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005441 EXPECT_EQ(
5442 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5443 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005444
mflodmancc3d4422017-08-03 08:27:51 -07005445 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005446}
5447
mflodmancc3d4422017-08-03 08:27:51 -07005448TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005449 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005450 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005451 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5452 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5453 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005454 const int kWidth = 640;
5455 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005456
asaperssonfab67072017-04-04 05:51:49 -07005457 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005458
5459 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005460 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005461
5462 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005463 EXPECT_TRUE_WAIT(
5464 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005465
sprangc5d62e22017-04-02 23:53:04 -07005466 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005467
asaperssonfab67072017-04-04 05:51:49 -07005468 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005469 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005470 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005471
5472 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005473 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005474
Henrik Boström2671dac2020-05-19 16:29:09 +02005475 EXPECT_TRUE_WAIT(
5476 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005477
mflodmancc3d4422017-08-03 08:27:51 -07005478 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005479}
5480
mflodmancc3d4422017-08-03 08:27:51 -07005481TEST_F(VideoStreamEncoderTest,
5482 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005483 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005485 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5486 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5487 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005488 const int kWidth = 640;
5489 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005490
5491 // We expect the n initial frames to get dropped.
5492 int i;
5493 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005494 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005495 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005496 }
5497 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005498 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005499 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005500
5501 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005502 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005503
mflodmancc3d4422017-08-03 08:27:51 -07005504 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005505}
5506
mflodmancc3d4422017-08-03 08:27:51 -07005507TEST_F(VideoStreamEncoderTest,
5508 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005509 const int kWidth = 640;
5510 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005512 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005513
5514 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005515 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005516 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005517
asaperssonfab67072017-04-04 05:51:49 -07005518 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005519 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005520 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005521
mflodmancc3d4422017-08-03 08:27:51 -07005522 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005523}
5524
mflodmancc3d4422017-08-03 08:27:51 -07005525TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005526 const int kWidth = 640;
5527 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005528 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005529
5530 VideoEncoderConfig video_encoder_config;
5531 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5532 // Make format different, to force recreation of encoder.
5533 video_encoder_config.video_format.parameters["foo"] = "foo";
5534 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005535 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005537 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005538
kthelgasonb83797b2017-02-14 11:57:25 -08005539 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005540 video_stream_encoder_->SetSource(&video_source_,
5541 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005542
asaperssonfab67072017-04-04 05:51:49 -07005543 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005544 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005545 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005546
mflodmancc3d4422017-08-03 08:27:51 -07005547 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005548 fake_encoder_.SetQualityScaling(true);
5549}
5550
Åsa Persson139f4dc2019-08-02 09:29:58 +02005551TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005552 webrtc::test::ScopedKeyValueConfig field_trials(
5553 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005554 "WebRTC-Video-QualityScalerSettings/"
5555 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5556 // Reset encoder for field trials to take effect.
5557 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005558 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5559 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005560 const int kWidth = 640;
5561 const int kHeight = 360;
5562
Henrik Boström381d1092020-05-12 18:49:07 +02005563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005564 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005565 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5566 // Frame should not be dropped.
5567 WaitForEncodedFrame(1);
5568
Henrik Boström381d1092020-05-12 18:49:07 +02005569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005570 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5571 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5572 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005573 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5574 // Frame should not be dropped.
5575 WaitForEncodedFrame(2);
5576
Henrik Boström381d1092020-05-12 18:49:07 +02005577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005578 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5579 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5580 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005581 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5582 // Expect to drop this frame, the wait should time out.
5583 ExpectDroppedFrame();
5584
5585 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005586 EXPECT_TRUE_WAIT(
5587 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005588 video_stream_encoder_->Stop();
5589}
5590
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005591TEST_F(VideoStreamEncoderTest,
5592 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005593 webrtc::test::ScopedKeyValueConfig field_trials(
5594 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005595 "WebRTC-Video-QualityScalerSettings/"
5596 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5597 fake_encoder_.SetQualityScaling(false);
5598 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005599 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5600 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005601 const int kWidth = 640;
5602 const int kHeight = 360;
5603
5604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005605 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005606 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5607 // Frame should not be dropped.
5608 WaitForEncodedFrame(1);
5609
5610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5611 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5612 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5613 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5614 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5615 // Frame should not be dropped.
5616 WaitForEncodedFrame(2);
5617
5618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5619 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5620 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5621 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5622 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5623 // Not dropped since quality scaling is disabled.
5624 WaitForEncodedFrame(3);
5625
5626 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005627 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005628 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5629
5630 video_stream_encoder_->Stop();
5631}
5632
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005633TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005634 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005635 // Set simulcast.
5636 ResetEncoder("VP8", 3, 1, 1, false);
5637 fake_encoder_.SetQualityScaling(true);
5638 const int kWidth = 1280;
5639 const int kHeight = 720;
5640 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005641 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005642 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5643 // Frame should not be dropped.
5644 WaitForEncodedFrame(1);
5645
5646 // Trigger QVGA "singlecast"
5647 // Update the config.
5648 VideoEncoderConfig video_encoder_config;
5649 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5650 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005651 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005652 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005653 "VP8", /*max qp*/ 56, /*screencast*/ false,
5654 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005655 for (auto& layer : video_encoder_config.simulcast_layers) {
5656 layer.num_temporal_layers = 1;
5657 layer.max_framerate = kDefaultFramerate;
5658 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005659 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005660 video_encoder_config.content_type =
5661 VideoEncoderConfig::ContentType::kRealtimeVideo;
5662
5663 video_encoder_config.simulcast_layers[0].active = true;
5664 video_encoder_config.simulcast_layers[1].active = false;
5665 video_encoder_config.simulcast_layers[2].active = false;
5666
5667 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5668 kMaxPayloadLength);
5669 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5670
5671 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5672 // Frame should not be dropped.
5673 WaitForEncodedFrame(2);
5674
5675 // Trigger HD "singlecast"
5676 video_encoder_config.simulcast_layers[0].active = false;
5677 video_encoder_config.simulcast_layers[1].active = false;
5678 video_encoder_config.simulcast_layers[2].active = true;
5679
5680 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5681 kMaxPayloadLength);
5682 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5683
5684 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5685 // Frame should be dropped because of initial frame drop.
5686 ExpectDroppedFrame();
5687
5688 // Expect the sink_wants to specify a scaled frame.
5689 EXPECT_TRUE_WAIT(
5690 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5691 video_stream_encoder_->Stop();
5692}
5693
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005694TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005695 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005696 // Set simulcast.
5697 ResetEncoder("VP9", 1, 1, 3, false);
5698 fake_encoder_.SetQualityScaling(true);
5699 const int kWidth = 1280;
5700 const int kHeight = 720;
5701 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005702 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005703 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5704 // Frame should not be dropped.
5705 WaitForEncodedFrame(1);
5706
5707 // Trigger QVGA "singlecast"
5708 // Update the config.
5709 VideoEncoderConfig video_encoder_config;
5710 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5711 &video_encoder_config);
5712 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5713 vp9_settings.numberOfSpatialLayers = 3;
5714 // Since only one layer is active - automatic resize should be enabled.
5715 vp9_settings.automaticResizeOn = true;
5716 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005717 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005718 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005719 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005720 video_encoder_config.content_type =
5721 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005722 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005723 // which SVC layers are active.
5724 video_encoder_config.simulcast_layers.resize(3);
5725
5726 video_encoder_config.simulcast_layers[0].active = true;
5727 video_encoder_config.simulcast_layers[1].active = false;
5728 video_encoder_config.simulcast_layers[2].active = false;
5729
5730 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5731 kMaxPayloadLength);
5732 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5733
5734 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5735 // Frame should not be dropped.
5736 WaitForEncodedFrame(2);
5737
5738 // Trigger HD "singlecast"
5739 video_encoder_config.simulcast_layers[0].active = false;
5740 video_encoder_config.simulcast_layers[1].active = false;
5741 video_encoder_config.simulcast_layers[2].active = true;
5742
5743 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5744 kMaxPayloadLength);
5745 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5746
5747 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5748 // Frame should be dropped because of initial frame drop.
5749 ExpectDroppedFrame();
5750
5751 // Expect the sink_wants to specify a scaled frame.
5752 EXPECT_TRUE_WAIT(
5753 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5754 video_stream_encoder_->Stop();
5755}
5756
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005757TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005758 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5759 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5760 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5761 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5762 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5763 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5764 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5765 fake_encoder_.SetResolutionBitrateLimits(
5766 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5767
5768 VideoEncoderConfig video_encoder_config;
5769 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5770 &video_encoder_config);
5771 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5772 vp9_settings.numberOfSpatialLayers = 3;
5773 // Since only one layer is active - automatic resize should be enabled.
5774 vp9_settings.automaticResizeOn = true;
5775 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005776 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005777 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005778 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005779 video_encoder_config.content_type =
5780 VideoEncoderConfig::ContentType::kRealtimeVideo;
5781 // Simulcast layers are used to indicate which spatial layers are active.
5782 video_encoder_config.simulcast_layers.resize(3);
5783 video_encoder_config.simulcast_layers[0].active = false;
5784 video_encoder_config.simulcast_layers[1].active = true;
5785 video_encoder_config.simulcast_layers[2].active = false;
5786
5787 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5788 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005789
5790 // The encoder bitrate limits for 360p should be used.
5791 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005792 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005793 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5794 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5795 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5796 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5797 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5798 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005799 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005801 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005803
5804 // The encoder bitrate limits for 270p should be used.
5805 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005806 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005807 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5808 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5809 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5810 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5811 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5812 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005813 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005814 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005815 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005816 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005817
5818 video_stream_encoder_->Stop();
5819}
5820
5821TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005822 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5823 VideoEncoderConfig video_encoder_config;
5824 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5825 &video_encoder_config);
5826 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5827 vp9_settings.numberOfSpatialLayers = 3;
5828 // Since only one layer is active - automatic resize should be enabled.
5829 vp9_settings.automaticResizeOn = true;
5830 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005831 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005832 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005833 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005834 video_encoder_config.content_type =
5835 VideoEncoderConfig::ContentType::kRealtimeVideo;
5836 // Simulcast layers are used to indicate which spatial layers are active.
5837 video_encoder_config.simulcast_layers.resize(3);
5838 video_encoder_config.simulcast_layers[0].active = false;
5839 video_encoder_config.simulcast_layers[1].active = true;
5840 video_encoder_config.simulcast_layers[2].active = false;
5841
5842 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5843 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005844
5845 // The default bitrate limits for 360p should be used.
5846 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005847 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5848 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005849 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005851 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5852 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5853 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5854 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5855 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5856 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005857 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005858 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005859 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005860 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005861
5862 // The default bitrate limits for 270p should be used.
5863 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005864 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5865 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005866 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005868 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5869 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5870 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5871 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5872 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5873 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005874 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005875 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005876 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005877 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005878
5879 video_stream_encoder_->Stop();
5880}
5881
5882TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005883 webrtc::test::ScopedKeyValueConfig field_trials(
5884 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005885 VideoEncoderConfig video_encoder_config;
5886 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5887 &video_encoder_config);
5888 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5889 vp9_settings.numberOfSpatialLayers = 3;
5890 // Since only one layer is active - automatic resize should be enabled.
5891 vp9_settings.automaticResizeOn = true;
5892 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005893 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005894 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005895 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005896 video_encoder_config.content_type =
5897 VideoEncoderConfig::ContentType::kRealtimeVideo;
5898 // Simulcast layers are used to indicate which spatial layers are active.
5899 video_encoder_config.simulcast_layers.resize(3);
5900 video_encoder_config.simulcast_layers[0].active = false;
5901 video_encoder_config.simulcast_layers[1].active = true;
5902 video_encoder_config.simulcast_layers[2].active = false;
5903
5904 // Reset encoder for field trials to take effect.
5905 ConfigureEncoder(video_encoder_config.Copy());
5906
5907 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5908 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005909
5910 // The default bitrate limits for 360p should not be used.
5911 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005912 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5913 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005914 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005915 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005916 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5917 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5918 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5919 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5920 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5921 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005922 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005923 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005924
5925 video_stream_encoder_->Stop();
5926}
5927
5928TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5929 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5930 /*num_spatial_layers=*/1, /*screenshare=*/false);
5931
5932 // The default singlecast bitrate limits for 720p should not be used.
5933 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005934 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5935 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005936 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005937 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005938 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5939 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5940 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5941 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5942 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5943 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005944 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005945 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005946
5947 video_stream_encoder_->Stop();
5948}
5949
5950TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005951 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5952 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5953 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5954 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5955 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5956 fake_encoder_.SetResolutionBitrateLimits(
5957 {kEncoderLimits180p, kEncoderLimits720p});
5958
5959 VideoEncoderConfig video_encoder_config;
5960 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5961 &video_encoder_config);
5962 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5963 vp9_settings.numberOfSpatialLayers = 3;
5964 // Since only one layer is active - automatic resize should be enabled.
5965 vp9_settings.automaticResizeOn = true;
5966 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005967 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005968 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005969 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005970 video_encoder_config.content_type =
5971 VideoEncoderConfig::ContentType::kRealtimeVideo;
5972 // Simulcast layers are used to indicate which spatial layers are active.
5973 video_encoder_config.simulcast_layers.resize(3);
5974 video_encoder_config.simulcast_layers[0].active = true;
5975 video_encoder_config.simulcast_layers[1].active = false;
5976 video_encoder_config.simulcast_layers[2].active = false;
5977
5978 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5979 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005980
5981 // Limits not applied on lowest stream, limits for 180p should not be used.
5982 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005983 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005984 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5985 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5986 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5987 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5988 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5989 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005990 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005991 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005992 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005993 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005994
5995 video_stream_encoder_->Stop();
5996}
5997
5998TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005999 InitialFrameDropActivatesWhenResolutionIncreases) {
6000 const int kWidth = 640;
6001 const int kHeight = 360;
6002
6003 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006004 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006005 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6006 // Frame should not be dropped.
6007 WaitForEncodedFrame(1);
6008
6009 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006010 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006011 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6012 // Frame should not be dropped, bitrate not too low for frame.
6013 WaitForEncodedFrame(2);
6014
6015 // Incoming resolution increases.
6016 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6017 // Expect to drop this frame, bitrate too low for frame.
6018 ExpectDroppedFrame();
6019
6020 // Expect the sink_wants to specify a scaled frame.
6021 EXPECT_TRUE_WAIT(
6022 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6023 video_stream_encoder_->Stop();
6024}
6025
6026TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6027 const int kWidth = 640;
6028 const int kHeight = 360;
6029 // So that quality scaling doesn't happen by itself.
6030 fake_encoder_.SetQp(kQpHigh);
6031
6032 AdaptingFrameForwarder source(&time_controller_);
6033 source.set_adaptation_enabled(true);
6034 video_stream_encoder_->SetSource(
6035 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6036
6037 int timestamp = 1;
6038
6039 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006040 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006041 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6042 WaitForEncodedFrame(timestamp);
6043 timestamp += 9000;
6044 // Long pause to disable all first BWE drop logic.
6045 AdvanceTime(TimeDelta::Millis(1000));
6046
6047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006048 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006049 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6050 // Not dropped frame, as initial frame drop is disabled by now.
6051 WaitForEncodedFrame(timestamp);
6052 timestamp += 9000;
6053 AdvanceTime(TimeDelta::Millis(100));
6054
6055 // Quality adaptation down.
6056 video_stream_encoder_->TriggerQualityLow();
6057
6058 // Adaptation has an effect.
6059 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6060 5000);
6061
6062 // Frame isn't dropped as initial frame dropper is disabled.
6063 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6064 WaitForEncodedFrame(timestamp);
6065 timestamp += 9000;
6066 AdvanceTime(TimeDelta::Millis(100));
6067
6068 // Quality adaptation up.
6069 video_stream_encoder_->TriggerQualityHigh();
6070
6071 // Adaptation has an effect.
6072 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6073 5000);
6074
6075 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6076 // Frame should not be dropped, as initial framedropper is off.
6077 WaitForEncodedFrame(timestamp);
6078
6079 video_stream_encoder_->Stop();
6080}
6081
Åsa Persson7f354f82021-02-04 15:52:15 +01006082TEST_F(VideoStreamEncoderTest,
6083 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6084 const int kMinStartBps360p = 222000;
6085 fake_encoder_.SetResolutionBitrateLimits(
6086 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6087 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6088 800000)});
6089
6090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6091 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6093 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6094 0, 0, 0);
6095 // Frame should not be dropped, bitrate not too low for frame.
6096 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6097 WaitForEncodedFrame(1);
6098
6099 // Incoming resolution increases, initial frame drop activates.
6100 // Frame should be dropped, link allocation too low for frame.
6101 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6102 ExpectDroppedFrame();
6103
6104 // Expect sink_wants to specify a scaled frame.
6105 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6106 5000);
6107 video_stream_encoder_->Stop();
6108}
6109
6110TEST_F(VideoStreamEncoderTest,
6111 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6112 const int kMinStartBps360p = 222000;
6113 fake_encoder_.SetResolutionBitrateLimits(
6114 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6115 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6116 800000)});
6117
6118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6119 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6120 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6121 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6122 0, 0, 0);
6123 // Frame should not be dropped, bitrate not too low for frame.
6124 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6125 WaitForEncodedFrame(1);
6126
6127 // Incoming resolution increases, initial frame drop activates.
6128 // Frame should be dropped, link allocation not too low for frame.
6129 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6130 WaitForEncodedFrame(2);
6131
6132 video_stream_encoder_->Stop();
6133}
6134
Åsa Perssone644a032019-11-08 15:56:00 +01006135TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006136 webrtc::test::ScopedKeyValueConfig field_trials(
6137 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006138 "WebRTC-Video-QualityRampupSettings/"
6139 "min_pixels:921600,min_duration_ms:2000/");
6140
6141 const int kWidth = 1280;
6142 const int kHeight = 720;
6143 const int kFps = 10;
6144 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006145
6146 // Reset encoder for field trials to take effect.
6147 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006148 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006149 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006150 ConfigureEncoder(std::move(config));
6151 fake_encoder_.SetQp(kQpLow);
6152
6153 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006154 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006155 source.set_adaptation_enabled(true);
6156 video_stream_encoder_->SetSource(&source,
6157 DegradationPreference::MAINTAIN_FRAMERATE);
6158
6159 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006160 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006161 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006162 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006163
6164 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006165 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006166 int64_t timestamp_ms = kFrameIntervalMs;
6167 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6168 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006169 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6170 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006171
6172 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6174 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006175
Artem Titovab30d722021-07-27 16:22:11 +02006176 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006177 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006178 for (size_t i = 1; i <= 10; i++) {
6179 timestamp_ms += kFrameIntervalMs;
6180 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6181 WaitForEncodedFrame(timestamp_ms);
6182 }
Åsa Persson06defc42021-09-10 15:28:48 +02006183
6184 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6185 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6186 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6187 timestamp_ms += kFrameIntervalMs;
6188 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6189 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006190 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6191 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6192
Åsa Persson06defc42021-09-10 15:28:48 +02006193 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006194 timestamp_ms += kFrameIntervalMs;
6195 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6196 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006197 // The ramp-up code involves the adaptation queue, give it time to execute.
6198 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006199 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006200 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006201
6202 // Frame should not be adapted.
6203 timestamp_ms += kFrameIntervalMs;
6204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6205 WaitForEncodedFrame(kWidth, kHeight);
6206 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6207
6208 video_stream_encoder_->Stop();
6209}
6210
mflodmancc3d4422017-08-03 08:27:51 -07006211TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006212 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006213 webrtc::test::ScopedKeyValueConfig field_trials(
6214 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006215 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006216 source.set_adaptation_enabled(true);
6217 video_stream_encoder_->SetSource(&source,
6218 DegradationPreference::MAINTAIN_FRAMERATE);
6219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006220 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006221 fake_encoder_.SetQp(kQpHigh + 1);
6222 const int kWidth = 1280;
6223 const int kHeight = 720;
6224 const int64_t kFrameIntervalMs = 100;
6225 int64_t timestamp_ms = kFrameIntervalMs;
6226 for (size_t i = 1; i <= 100; i++) {
6227 timestamp_ms += kFrameIntervalMs;
6228 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6229 WaitForEncodedFrame(timestamp_ms);
6230 }
6231 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6232 // for the first time.
6233 // TODO(eshr): We should avoid these waits by using threads with simulated
6234 // time.
6235 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6236 2000 * 2.5 * 2);
6237 timestamp_ms += kFrameIntervalMs;
6238 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6239 WaitForEncodedFrame(timestamp_ms);
6240 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6241 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6242 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6243
6244 // Disable Quality scaling by turning off scaler on the encoder and
6245 // reconfiguring.
6246 fake_encoder_.SetQualityScaling(false);
6247 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6248 kMaxPayloadLength);
6249 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006250 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006251 // Since we turned off the quality scaler, the adaptations made by it are
6252 // removed.
6253 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6254 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6255
6256 video_stream_encoder_->Stop();
6257}
6258
6259TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006260 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6261 const int kTooSmallWidth = 10;
6262 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006263 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006264 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006265
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006266 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006267 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006268 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006269 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006270 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006271 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6272
6273 // Trigger adapt down, too small frame, expect no change.
6274 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006275 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006276 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006277 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006278 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6279 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6280
mflodmancc3d4422017-08-03 08:27:51 -07006281 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006282}
6283
mflodmancc3d4422017-08-03 08:27:51 -07006284TEST_F(VideoStreamEncoderTest,
6285 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006286 const int kTooSmallWidth = 10;
6287 const int kTooSmallHeight = 10;
6288 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006289 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006290 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006291
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006292 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006293 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006294 video_stream_encoder_->SetSource(&source,
6295 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006296 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6299
6300 // Trigger adapt down, expect limited framerate.
6301 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006302 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006303 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006304 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006305 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6306 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6307 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6308
6309 // Trigger adapt down, too small frame, expect no change.
6310 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006311 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006312 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006313 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6316 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6317
mflodmancc3d4422017-08-03 08:27:51 -07006318 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006319}
6320
mflodmancc3d4422017-08-03 08:27:51 -07006321TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006322 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006323 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006324 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006325 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006326 const int kFrameWidth = 1280;
6327 const int kFrameHeight = 720;
6328 video_source_.IncomingCapturedFrame(
6329 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006330 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006331 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006332}
6333
sprangb1ca0732017-02-01 08:38:12 -08006334// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006335TEST_F(VideoStreamEncoderTest,
6336 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006337 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006338 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006339
6340 const int kFrameWidth = 1280;
6341 const int kFrameHeight = 720;
6342 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006343 // requested by
6344 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006345 video_source_.set_adaptation_enabled(true);
6346
6347 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006348 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006349 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006350
6351 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006352 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006353 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006354 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006355 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006356
asaperssonfab67072017-04-04 05:51:49 -07006357 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006358 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006359 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006360 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006361 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006362
mflodmancc3d4422017-08-03 08:27:51 -07006363 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006364}
sprangfe627f32017-03-29 08:24:59 -07006365
mflodmancc3d4422017-08-03 08:27:51 -07006366TEST_F(VideoStreamEncoderTest,
6367 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006368 const int kFrameWidth = 1280;
6369 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006370
Henrik Boström381d1092020-05-12 18:49:07 +02006371 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006372 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006374 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006375 video_source_.set_adaptation_enabled(true);
6376
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006377 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006378
6379 video_source_.IncomingCapturedFrame(
6380 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006381 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006382
6383 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006385
6386 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006387 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006388 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006391 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006392 }
6393
6394 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006395 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006396 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006397 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006398 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006399 video_source_.IncomingCapturedFrame(
6400 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006401 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006402 ++num_frames_dropped;
6403 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006404 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006405 }
6406 }
6407
sprang4847ae62017-06-27 07:06:52 -07006408 // Add some slack to account for frames dropped by the frame dropper.
6409 const int kErrorMargin = 1;
6410 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006411 kErrorMargin);
6412
6413 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006415 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006416 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006417 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006418 video_source_.IncomingCapturedFrame(
6419 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006420 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006421 ++num_frames_dropped;
6422 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006423 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006424 }
6425 }
sprang4847ae62017-06-27 07:06:52 -07006426 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006427 kErrorMargin);
6428
6429 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006430 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006431 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006432 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006433 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006434 video_source_.IncomingCapturedFrame(
6435 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006436 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006437 ++num_frames_dropped;
6438 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006439 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006440 }
6441 }
sprang4847ae62017-06-27 07:06:52 -07006442 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006443 kErrorMargin);
6444
6445 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006446 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006447 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006448 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006449 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006450 video_source_.IncomingCapturedFrame(
6451 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006452 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006453 ++num_frames_dropped;
6454 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006455 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006456 }
6457 }
6458 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6459
mflodmancc3d4422017-08-03 08:27:51 -07006460 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006461}
6462
mflodmancc3d4422017-08-03 08:27:51 -07006463TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006464 const int kFramerateFps = 5;
6465 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006466 const int kFrameWidth = 1280;
6467 const int kFrameHeight = 720;
6468
sprang4847ae62017-06-27 07:06:52 -07006469 // Reconfigure encoder with two temporal layers and screensharing, which will
6470 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006471 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006472
Henrik Boström381d1092020-05-12 18:49:07 +02006473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006474 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006475 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006476 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006477 video_source_.set_adaptation_enabled(true);
6478
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006479 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006480
6481 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006482 rtc::VideoSinkWants last_wants;
6483 do {
6484 last_wants = video_source_.sink_wants();
6485
sprangc5d62e22017-04-02 23:53:04 -07006486 // Insert frames to get a new fps estimate...
6487 for (int j = 0; j < kFramerateFps; ++j) {
6488 video_source_.IncomingCapturedFrame(
6489 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006490 if (video_source_.last_sent_width()) {
6491 sink_.WaitForEncodedFrame(timestamp_ms);
6492 }
sprangc5d62e22017-04-02 23:53:04 -07006493 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006494 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006495 }
6496 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006497 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006498 } while (video_source_.sink_wants().max_framerate_fps <
6499 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006500
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006501 EXPECT_THAT(video_source_.sink_wants(),
6502 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006503
mflodmancc3d4422017-08-03 08:27:51 -07006504 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006505}
asaperssonf7e294d2017-06-13 23:25:22 -07006506
mflodmancc3d4422017-08-03 08:27:51 -07006507TEST_F(VideoStreamEncoderTest,
6508 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006509 const int kWidth = 1280;
6510 const int kHeight = 720;
6511 const int64_t kFrameIntervalMs = 150;
6512 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006513 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006514 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006515
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006516 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006517 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006518 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006519 video_stream_encoder_->SetSource(&source,
6520 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006521 timestamp_ms += kFrameIntervalMs;
6522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006523 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006524 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6527 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6528
6529 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006530 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006531 timestamp_ms += kFrameIntervalMs;
6532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006534 EXPECT_THAT(source.sink_wants(),
6535 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6538 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6539
6540 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006541 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006542 timestamp_ms += kFrameIntervalMs;
6543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006544 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006545 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
6550 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006555 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6569
6570 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006571 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006572 timestamp_ms += kFrameIntervalMs;
6573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006575 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6578 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6579
6580 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006581 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006582 timestamp_ms += kFrameIntervalMs;
6583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006584 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006585 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6589
6590 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006591 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006595 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006596 rtc::VideoSinkWants last_wants = source.sink_wants();
6597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6599 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6600
6601 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006602 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006603 timestamp_ms += kFrameIntervalMs;
6604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006605 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006606 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6609 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6610
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006611 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006612 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006613 timestamp_ms += kFrameIntervalMs;
6614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006616 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6619 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6620
6621 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006623 timestamp_ms += kFrameIntervalMs;
6624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006626 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6629 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6630
6631 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006632 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006633 timestamp_ms += kFrameIntervalMs;
6634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006635 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006636 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6639 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6640
6641 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006642 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006643 timestamp_ms += kFrameIntervalMs;
6644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006645 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006646 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6649 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6650
6651 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006652 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006653 timestamp_ms += kFrameIntervalMs;
6654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006656 EXPECT_THAT(source.sink_wants(), FpsMax());
6657 EXPECT_EQ(source.sink_wants().max_pixel_count,
6658 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006659 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6661 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6662
6663 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006664 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006665 timestamp_ms += kFrameIntervalMs;
6666 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006667 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006668 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006669 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6671 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6672
Åsa Persson30ab0152019-08-27 12:22:33 +02006673 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006674 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006675 timestamp_ms += kFrameIntervalMs;
6676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006677 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006678 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6682 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6683
6684 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006685 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006686 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006687 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6688
mflodmancc3d4422017-08-03 08:27:51 -07006689 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006690}
6691
mflodmancc3d4422017-08-03 08:27:51 -07006692TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006693 const int kWidth = 1280;
6694 const int kHeight = 720;
6695 const int64_t kFrameIntervalMs = 150;
6696 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006698 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006699
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006700 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006701 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006702 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006703 video_stream_encoder_->SetSource(&source,
6704 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006705 timestamp_ms += kFrameIntervalMs;
6706 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006707 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006708 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6713 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6714 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6715
6716 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006717 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006718 timestamp_ms += kFrameIntervalMs;
6719 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006720 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006721 EXPECT_THAT(source.sink_wants(),
6722 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6724 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6725 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6726 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6727 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6728 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6729
6730 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006731 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006732 timestamp_ms += kFrameIntervalMs;
6733 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006734 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006735 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006736 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6738 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6739 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6740 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6741 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6742
6743 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006744 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006745 timestamp_ms += kFrameIntervalMs;
6746 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006747 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006748 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006750 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6751 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6753 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6754 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6755
Evan Shrubsole64469032020-06-11 10:45:29 +02006756 // Trigger cpu adapt up, expect no change since QP is most limited.
6757 {
6758 // Store current sink wants since we expect no change and if there is no
6759 // change then last_wants() is not updated.
6760 auto previous_sink_wants = source.sink_wants();
6761 video_stream_encoder_->TriggerCpuUnderuse();
6762 timestamp_ms += kFrameIntervalMs;
6763 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6764 WaitForEncodedFrame(timestamp_ms);
6765 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6766 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6767 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6768 }
6769
6770 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6771 video_stream_encoder_->TriggerQualityHigh();
6772 timestamp_ms += kFrameIntervalMs;
6773 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6774 WaitForEncodedFrame(timestamp_ms);
6775 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6776 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6777 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6778 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6779 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6780 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6781 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6782
6783 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6784 // expect increased resolution (960x540@30fps).
6785 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006786 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006787 timestamp_ms += kFrameIntervalMs;
6788 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006789 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006790 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006791 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6792 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6793 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6794 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6795 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006796 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006797
Evan Shrubsole64469032020-06-11 10:45:29 +02006798 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6799 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006800 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006801 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006802 timestamp_ms += kFrameIntervalMs;
6803 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006804 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006806 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6808 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6810 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6811 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006812 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006813
6814 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006815 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006816 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006817 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006818 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006819
mflodmancc3d4422017-08-03 08:27:51 -07006820 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006821}
6822
mflodmancc3d4422017-08-03 08:27:51 -07006823TEST_F(VideoStreamEncoderTest,
6824 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006825 const int kWidth = 640;
6826 const int kHeight = 360;
6827 const int kFpsLimit = 15;
6828 const int64_t kFrameIntervalMs = 150;
6829 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006831 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006832
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006833 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006834 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006835 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006836 video_stream_encoder_->SetSource(&source,
6837 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006838 timestamp_ms += kFrameIntervalMs;
6839 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006840 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006841 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6846 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6848
6849 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006850 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006851 timestamp_ms += kFrameIntervalMs;
6852 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006853 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006854 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6857 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6858 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6859 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6860 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6861
6862 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006863 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006864 timestamp_ms += kFrameIntervalMs;
6865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006866 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006867 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006868 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006870 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6871 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6872 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6874
Evan Shrubsole64469032020-06-11 10:45:29 +02006875 // Trigger cpu adapt up, expect no change because quality is most limited.
6876 {
6877 auto previous_sink_wants = source.sink_wants();
6878 // Store current sink wants since we expect no change ind if there is no
6879 // change then last__wants() is not updated.
6880 video_stream_encoder_->TriggerCpuUnderuse();
6881 timestamp_ms += kFrameIntervalMs;
6882 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6883 WaitForEncodedFrame(timestamp_ms);
6884 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6885 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6886 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6887 }
6888
6889 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6890 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006891 timestamp_ms += kFrameIntervalMs;
6892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006893 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006894 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006895 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6896 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6897 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006898 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6899 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6900 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006901
Evan Shrubsole64469032020-06-11 10:45:29 +02006902 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006903 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006904 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006905 timestamp_ms += kFrameIntervalMs;
6906 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006907 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006908 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006909 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6911 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6912 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6913 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006914 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006915
6916 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006917 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006919 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006920 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006921
mflodmancc3d4422017-08-03 08:27:51 -07006922 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006923}
6924
mflodmancc3d4422017-08-03 08:27:51 -07006925TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006926 const int kFrameWidth = 1920;
6927 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006928 // 2/3 of 1920.
6929 const int kAdaptedFrameWidth = 1280;
6930 // 2/3 of 1080.
6931 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006932 const int kFramerate = 24;
6933
Henrik Boström381d1092020-05-12 18:49:07 +02006934 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006935 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006936 // Trigger reconfigure encoder (without resetting the entire instance).
6937 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006938 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6939 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006940 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006941 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006942 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006943 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006944 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006945 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006946
6947 video_source_.set_adaptation_enabled(true);
6948
6949 video_source_.IncomingCapturedFrame(
6950 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006951 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006952
6953 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006954 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006955 video_source_.IncomingCapturedFrame(
6956 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006957 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006958
mflodmancc3d4422017-08-03 08:27:51 -07006959 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006960}
6961
mflodmancc3d4422017-08-03 08:27:51 -07006962TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006963 const int kFrameWidth = 1280;
6964 const int kFrameHeight = 720;
6965 const int kLowFps = 2;
6966 const int kHighFps = 30;
6967
Henrik Boström381d1092020-05-12 18:49:07 +02006968 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006969 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006970
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006971 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006972 max_framerate_ = kLowFps;
6973
6974 // Insert 2 seconds of 2fps video.
6975 for (int i = 0; i < kLowFps * 2; ++i) {
6976 video_source_.IncomingCapturedFrame(
6977 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6978 WaitForEncodedFrame(timestamp_ms);
6979 timestamp_ms += 1000 / kLowFps;
6980 }
6981
6982 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006985 video_source_.IncomingCapturedFrame(
6986 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6987 WaitForEncodedFrame(timestamp_ms);
6988 timestamp_ms += 1000 / kLowFps;
6989
6990 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6991
6992 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006993 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
Markus Handell2cfc1af2022-08-19 08:16:48 +00006994 constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
sprang4847ae62017-06-27 07:06:52 -07006995 max_framerate_ = kHighFps;
6996 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6997 video_source_.IncomingCapturedFrame(
6998 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6999 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7000 // be dropped if the encoder hans't been updated with the new higher target
7001 // framerate yet, causing it to overshoot the target bitrate and then
7002 // suffering the wrath of the media optimizer.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007003 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
7004 timestamp_ms += kFrameInterval.ms();
sprang4847ae62017-06-27 07:06:52 -07007005 }
7006
7007 // Don expect correct measurement just yet, but it should be higher than
7008 // before.
7009 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7010
mflodmancc3d4422017-08-03 08:27:51 -07007011 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007012}
7013
mflodmancc3d4422017-08-03 08:27:51 -07007014TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007015 const int kFrameWidth = 1280;
7016 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007017 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007018 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007019 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007020
Henrik Boström381d1092020-05-12 18:49:07 +02007021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007023 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007024
7025 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007026 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007027 video_source_.IncomingCapturedFrame(
7028 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7029 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007030 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007031
7032 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007034 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007035
7036 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007037 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007038 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007039
Per Kjellanderdcef6412020-10-07 15:09:05 +02007040 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007041 video_source_.IncomingCapturedFrame(
7042 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7043 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007044 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007045
mflodmancc3d4422017-08-03 08:27:51 -07007046 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007047}
ilnik6b826ef2017-06-16 06:53:48 -07007048
Niels Möller4db138e2018-04-19 09:04:13 +02007049TEST_F(VideoStreamEncoderTest,
7050 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7051 const int kFrameWidth = 1280;
7052 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007053 const test::ScopedKeyValueConfig kFieldTrials;
7054 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007056 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007057 video_source_.IncomingCapturedFrame(
7058 CreateFrame(1, kFrameWidth, kFrameHeight));
7059 WaitForEncodedFrame(1);
7060 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7061 .low_encode_usage_threshold_percent,
7062 default_options.low_encode_usage_threshold_percent);
7063 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7064 .high_encode_usage_threshold_percent,
7065 default_options.high_encode_usage_threshold_percent);
7066 video_stream_encoder_->Stop();
7067}
7068
7069TEST_F(VideoStreamEncoderTest,
7070 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7071 const int kFrameWidth = 1280;
7072 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007073 const test::ScopedKeyValueConfig kFieldTrials;
7074 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007075 hardware_options.low_encode_usage_threshold_percent = 150;
7076 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007077 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007078
Henrik Boström381d1092020-05-12 18:49:07 +02007079 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007080 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007081 video_source_.IncomingCapturedFrame(
7082 CreateFrame(1, kFrameWidth, kFrameHeight));
7083 WaitForEncodedFrame(1);
7084 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7085 .low_encode_usage_threshold_percent,
7086 hardware_options.low_encode_usage_threshold_percent);
7087 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7088 .high_encode_usage_threshold_percent,
7089 hardware_options.high_encode_usage_threshold_percent);
7090 video_stream_encoder_->Stop();
7091}
7092
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007093TEST_F(VideoStreamEncoderTest,
7094 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7095 const int kFrameWidth = 1280;
7096 const int kFrameHeight = 720;
7097
Markus Handell8e4197b2022-05-30 15:45:28 +02007098 const test::ScopedKeyValueConfig kFieldTrials;
7099 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007101 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007102 video_source_.IncomingCapturedFrame(
7103 CreateFrame(1, kFrameWidth, kFrameHeight));
7104 WaitForEncodedFrame(1);
7105 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7106 .low_encode_usage_threshold_percent,
7107 default_options.low_encode_usage_threshold_percent);
7108 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7109 .high_encode_usage_threshold_percent,
7110 default_options.high_encode_usage_threshold_percent);
7111
Markus Handell8e4197b2022-05-30 15:45:28 +02007112 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007113 hardware_options.low_encode_usage_threshold_percent = 150;
7114 hardware_options.high_encode_usage_threshold_percent = 200;
7115 fake_encoder_.SetIsHardwareAccelerated(true);
7116
7117 video_source_.IncomingCapturedFrame(
7118 CreateFrame(2, kFrameWidth, kFrameHeight));
7119 WaitForEncodedFrame(2);
7120
7121 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7122 .low_encode_usage_threshold_percent,
7123 hardware_options.low_encode_usage_threshold_percent);
7124 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7125 .high_encode_usage_threshold_percent,
7126 hardware_options.high_encode_usage_threshold_percent);
7127
7128 video_stream_encoder_->Stop();
7129}
7130
Niels Möller6bb5ab92019-01-11 11:11:10 +01007131TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7132 const int kFrameWidth = 320;
7133 const int kFrameHeight = 240;
7134 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007135 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007136 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7137
Henrik Boström381d1092020-05-12 18:49:07 +02007138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007139 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007140
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007141 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007142 max_framerate_ = kFps;
7143
7144 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7145 fake_encoder_.SimulateOvershoot(1.0);
7146 int num_dropped = 0;
7147 for (int i = 0; i < kNumFramesInRun; ++i) {
7148 video_source_.IncomingCapturedFrame(
7149 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7150 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007151 if (!TimedWaitForEncodedFrame(timestamp_ms,
7152 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007153 ++num_dropped;
7154 }
7155 timestamp_ms += 1000 / kFps;
7156 }
7157
Erik Språnga8d48ab2019-02-08 14:17:40 +01007158 // Framerate should be measured to be near the expected target rate.
7159 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7160
7161 // Frame drops should be within 5% of expected 0%.
7162 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007163
7164 // Make encoder produce frames at double the expected bitrate during 3 seconds
7165 // of video, verify number of drops. Rate needs to be slightly changed in
7166 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007167 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007168 const RateControlSettings trials =
7169 RateControlSettings::ParseFromFieldTrials();
7170 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007171 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007172 // frame dropping since the adjuter will try to just lower the target
7173 // bitrate rather than drop frames. If network headroom can be used, it
7174 // doesn't push back as hard so we don't need quite as much overshoot.
7175 // These numbers are unfortunately a bit magical but there's not trivial
7176 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007177 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007178 }
7179 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007181 kTargetBitrate + DataRate::KilobitsPerSec(1),
7182 kTargetBitrate + DataRate::KilobitsPerSec(1),
7183 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007184 num_dropped = 0;
7185 for (int i = 0; i < kNumFramesInRun; ++i) {
7186 video_source_.IncomingCapturedFrame(
7187 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7188 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007189 if (!TimedWaitForEncodedFrame(timestamp_ms,
7190 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007191 ++num_dropped;
7192 }
7193 timestamp_ms += 1000 / kFps;
7194 }
7195
Henrik Boström381d1092020-05-12 18:49:07 +02007196 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007197 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007198
7199 // Target framerate should be still be near the expected target, despite
7200 // the frame drops.
7201 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7202
7203 // Frame drops should be within 5% of expected 50%.
7204 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007205
7206 video_stream_encoder_->Stop();
7207}
7208
7209TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7210 const int kFrameWidth = 320;
7211 const int kFrameHeight = 240;
7212 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007213 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007214
7215 ASSERT_GT(max_framerate_, kActualInputFps);
7216
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007217 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007218 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007219 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007220 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007221
7222 // Insert 3 seconds of video, with an input fps lower than configured max.
7223 for (int i = 0; i < kActualInputFps * 3; ++i) {
7224 video_source_.IncomingCapturedFrame(
7225 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7226 // Wait up to two frame durations for a frame to arrive.
7227 WaitForEncodedFrame(timestamp_ms);
7228 timestamp_ms += 1000 / kActualInputFps;
7229 }
7230
7231 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7232
7233 video_stream_encoder_->Stop();
7234}
7235
Markus Handell9a478b52021-11-18 16:07:01 +01007236TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007237 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007238 test::FrameForwarder source;
7239 video_stream_encoder_->SetSource(&source,
7240 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007241 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007242 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007243
Markus Handell9a478b52021-11-18 16:07:01 +01007244 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007245 WaitForEncodedFrame(1);
7246 // On the very first frame full update should be forced.
7247 rect = fake_encoder_.GetLastUpdateRect();
7248 EXPECT_EQ(rect.offset_x, 0);
7249 EXPECT_EQ(rect.offset_y, 0);
7250 EXPECT_EQ(rect.height, codec_height_);
7251 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007252 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7253 // scheduled for processing during encoder queue processing of frame 2.
7254 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7255 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007256 WaitForEncodedFrame(3);
7257 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7258 rect = fake_encoder_.GetLastUpdateRect();
7259 EXPECT_EQ(rect.offset_x, 1);
7260 EXPECT_EQ(rect.offset_y, 0);
7261 EXPECT_EQ(rect.width, 10);
7262 EXPECT_EQ(rect.height, 1);
7263
Markus Handell9a478b52021-11-18 16:07:01 +01007264 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007265 WaitForEncodedFrame(4);
7266 // Previous frame was encoded, so no accumulation should happen.
7267 rect = fake_encoder_.GetLastUpdateRect();
7268 EXPECT_EQ(rect.offset_x, 0);
7269 EXPECT_EQ(rect.offset_y, 0);
7270 EXPECT_EQ(rect.width, 1);
7271 EXPECT_EQ(rect.height, 1);
7272
7273 video_stream_encoder_->Stop();
7274}
7275
Erik Språngd7329ca2019-02-21 21:19:53 +01007276TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007278 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007279
7280 // First frame is always keyframe.
7281 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7282 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007283 EXPECT_THAT(
7284 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007285 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007286
7287 // Insert delta frame.
7288 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7289 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007290 EXPECT_THAT(
7291 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007292 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007293
7294 // Request next frame be a key-frame.
7295 video_stream_encoder_->SendKeyFrame();
7296 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7297 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007298 EXPECT_THAT(
7299 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007300 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007301
7302 video_stream_encoder_->Stop();
7303}
7304
7305TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7306 // Setup simulcast with three streams.
7307 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007308 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007309 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7310 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007311 // Wait for all three layers before triggering event.
7312 sink_.SetNumExpectedLayers(3);
7313
7314 // First frame is always keyframe.
7315 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7316 WaitForEncodedFrame(1);
7317 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007318 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7319 VideoFrameType::kVideoFrameKey,
7320 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007321
7322 // Insert delta frame.
7323 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7324 WaitForEncodedFrame(2);
7325 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007326 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7327 VideoFrameType::kVideoFrameDelta,
7328 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007329
7330 // Request next frame be a key-frame.
7331 // Only first stream is configured to produce key-frame.
7332 video_stream_encoder_->SendKeyFrame();
7333 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7334 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007335
7336 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7337 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007338 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007339 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007340 VideoFrameType::kVideoFrameKey,
7341 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007342
7343 video_stream_encoder_->Stop();
7344}
7345
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007346TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007347 // SPS contains VUI with restrictions on the maximum number of reordered
7348 // pictures, there is no need to rewrite the bitstream to enable faster
7349 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007350 ResetEncoder("H264", 1, 1, 1, false);
7351
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007352 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007353 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007354 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007355
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007356 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007357 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007358
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007359 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7360 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007361
7362 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007363 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007364
7365 video_stream_encoder_->Stop();
7366}
7367
7368TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007369 // SPS does not contain VUI, the bitstream is will be rewritten with added
7370 // VUI with restrictions on the maximum number of reordered pictures to
7371 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007372 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7373 0x00, 0x00, 0x03, 0x03, 0xF4,
7374 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007375 ResetEncoder("H264", 1, 1, 1, false);
7376
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007378 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007379 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007380
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007381 fake_encoder_.SetEncodedImageData(
7382 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007383
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007384 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7385 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007386
7387 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007388 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007389
7390 video_stream_encoder_->Stop();
7391}
7392
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007393TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7394 const int kFrameWidth = 1280;
7395 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007396 const DataRate kTargetBitrate =
7397 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007398
Henrik Boström381d1092020-05-12 18:49:07 +02007399 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007400 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7402
7403 // Insert a first video frame. It should be dropped because of downscale in
7404 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007405 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007406 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7407 frame.set_rotation(kVideoRotation_270);
7408 video_source_.IncomingCapturedFrame(frame);
7409
7410 ExpectDroppedFrame();
7411
7412 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007413 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007414 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7415 frame.set_rotation(kVideoRotation_90);
7416 video_source_.IncomingCapturedFrame(frame);
7417
7418 WaitForEncodedFrame(timestamp_ms);
7419 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7420
7421 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007422 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007423 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7424 frame.set_rotation(kVideoRotation_180);
7425 video_source_.IncomingCapturedFrame(frame);
7426
7427 WaitForEncodedFrame(timestamp_ms);
7428 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7429
7430 video_stream_encoder_->Stop();
7431}
7432
Erik Språng5056af02019-09-02 15:53:11 +02007433TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7434 const int kFrameWidth = 320;
7435 const int kFrameHeight = 180;
7436
7437 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007439 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7440 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7441 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007442 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007443 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007444 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007445
7446 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007447 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007448 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7449 frame.set_rotation(kVideoRotation_270);
7450 video_source_.IncomingCapturedFrame(frame);
7451 WaitForEncodedFrame(timestamp_ms);
7452
7453 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007454 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007455 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7456 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007458 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007459 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007460 /*link_allocation=*/target_rate,
7461 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007462 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007463 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007464 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7465
7466 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7467 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7468 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007469 DataRate allocation_sum =
7470 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007471 EXPECT_EQ(min_rate, allocation_sum);
7472 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7473
7474 video_stream_encoder_->Stop();
7475}
7476
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007477TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007479 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007480 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007481 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007482 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7483 WaitForEncodedFrame(1);
7484
7485 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7486 ASSERT_TRUE(prev_rate_settings.has_value());
7487 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7488 kDefaultFramerate);
7489
7490 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7491 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7492 timestamp_ms += 1000 / kDefaultFramerate;
7493 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7494 WaitForEncodedFrame(timestamp_ms);
7495 }
7496 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7497 kDefaultFramerate);
7498 // Capture larger frame to trigger a reconfigure.
7499 codec_height_ *= 2;
7500 codec_width_ *= 2;
7501 timestamp_ms += 1000 / kDefaultFramerate;
7502 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7503 WaitForEncodedFrame(timestamp_ms);
7504
7505 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7506 auto current_rate_settings =
7507 fake_encoder_.GetAndResetLastRateControlSettings();
7508 // Ensure we have actually reconfigured twice
7509 // The rate settings should have been set again even though
7510 // they haven't changed.
7511 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007512 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007513
7514 video_stream_encoder_->Stop();
7515}
7516
philipeld9cc8c02019-09-16 14:53:40 +02007517struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007518 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007519 MOCK_METHOD(void,
7520 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007521 (const webrtc::SdpVideoFormat& format,
7522 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007523 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007524};
7525
philipel9b058032020-02-10 11:30:00 +01007526TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7527 constexpr int kDontCare = 100;
7528 StrictMock<MockEncoderSelector> encoder_selector;
7529 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7530 &fake_encoder_, &encoder_selector);
7531 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7532
7533 // Reset encoder for new configuration to take effect.
7534 ConfigureEncoder(video_encoder_config_.Copy());
7535
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007536 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007537
7538 video_source_.IncomingCapturedFrame(
7539 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007540 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007541 video_stream_encoder_->Stop();
7542
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007543 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007544 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007545 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7546 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007547 video_stream_encoder_.reset();
7548}
7549
7550TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7551 constexpr int kDontCare = 100;
7552
7553 NiceMock<MockEncoderSelector> encoder_selector;
7554 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7555 video_send_config_.encoder_settings.encoder_switch_request_callback =
7556 &switch_callback;
7557 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7558 &fake_encoder_, &encoder_selector);
7559 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7560
7561 // Reset encoder for new configuration to take effect.
7562 ConfigureEncoder(video_encoder_config_.Copy());
7563
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007564 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007565 .WillByDefault(Return(SdpVideoFormat("AV1")));
7566 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007567 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7568 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007569
Henrik Boström381d1092020-05-12 18:49:07 +02007570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007571 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7572 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7573 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007574 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007575 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007576 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007577 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007578
7579 video_stream_encoder_->Stop();
7580}
7581
philipel6daa3042022-04-11 10:48:28 +02007582TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7583 NiceMock<MockEncoderSelector> encoder_selector;
7584 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7585 video_send_config_.encoder_settings.encoder_switch_request_callback =
7586 &switch_callback;
7587 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7588 &fake_encoder_, &encoder_selector);
7589 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7590
7591 // Reset encoder for new configuration to take effect.
7592 ConfigureEncoder(video_encoder_config_.Copy());
7593
7594 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7595 .WillOnce(Return(absl::nullopt));
7596 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7597 .WillOnce(Return(SdpVideoFormat("AV1")));
7598 EXPECT_CALL(switch_callback,
7599 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7600 /*allow_default_fallback=*/false));
7601
7602 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7603 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7604 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7605 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7606 /*fraction_lost=*/0,
7607 /*round_trip_time_ms=*/0,
7608 /*cwnd_reduce_ratio=*/0);
7609
7610 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7611 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7612 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7613
7614 AdvanceTime(TimeDelta::Zero());
7615
7616 video_stream_encoder_->Stop();
7617}
7618
philipel9b058032020-02-10 11:30:00 +01007619TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7620 constexpr int kSufficientBitrateToNotDrop = 1000;
7621 constexpr int kDontCare = 100;
7622
7623 NiceMock<MockVideoEncoder> video_encoder;
7624 NiceMock<MockEncoderSelector> encoder_selector;
7625 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7626 video_send_config_.encoder_settings.encoder_switch_request_callback =
7627 &switch_callback;
7628 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7629 &video_encoder, &encoder_selector);
7630 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7631
7632 // Reset encoder for new configuration to take effect.
7633 ConfigureEncoder(video_encoder_config_.Copy());
7634
7635 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7636 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7637 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007638 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007639 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7640 /*stable_target_bitrate=*/
7641 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7642 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007643 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007644 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007645 /*cwnd_reduce_ratio=*/0);
7646
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007647 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007648 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007649 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007650 .WillByDefault(Return(SdpVideoFormat("AV2")));
7651
7652 rtc::Event encode_attempted;
7653 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007654 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7655 /*allow_default_fallback=*/true))
7656 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007657
7658 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007659 encode_attempted.Wait(TimeDelta::Seconds(3));
philipel9b058032020-02-10 11:30:00 +01007660
Markus Handell28c71802021-11-08 10:11:55 +01007661 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007662
philipel9b058032020-02-10 11:30:00 +01007663 video_stream_encoder_->Stop();
7664
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007665 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7666 // to it's factory, so in order for the encoder instance in the
7667 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7668 // reset the `video_stream_encoder_` here.
7669 video_stream_encoder_.reset();
7670}
7671
7672TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7673 NiceMock<MockVideoEncoder> video_encoder;
7674 NiceMock<MockEncoderSelector> encoder_selector;
7675 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7676 video_send_config_.encoder_settings.encoder_switch_request_callback =
7677 &switch_callback;
7678 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7679 &video_encoder, &encoder_selector);
7680 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7681
7682 // Reset encoder for new configuration to take effect.
7683 ConfigureEncoder(video_encoder_config_.Copy());
7684
7685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7686 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7687 /*round_trip_time_ms=*/0,
7688 /*cwnd_reduce_ratio=*/0);
7689 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7690
7691 ON_CALL(video_encoder, InitEncode(_, _))
7692 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7693 ON_CALL(encoder_selector, OnEncoderBroken)
7694 .WillByDefault(Return(SdpVideoFormat("AV2")));
7695
7696 rtc::Event encode_attempted;
7697 EXPECT_CALL(switch_callback,
7698 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7699 /*allow_default_fallback=*/true))
7700 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7701
7702 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007703 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007704
7705 AdvanceTime(TimeDelta::Zero());
7706
7707 video_stream_encoder_->Stop();
7708
7709 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7710 // to it's factory, so in order for the encoder instance in the
7711 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7712 // reset the `video_stream_encoder_` here.
7713 video_stream_encoder_.reset();
7714}
7715
7716TEST_F(VideoStreamEncoderTest,
7717 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7718 NiceMock<MockVideoEncoder> video_encoder;
7719 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7720 video_send_config_.encoder_settings.encoder_switch_request_callback =
7721 &switch_callback;
7722 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7723 &video_encoder, /*encoder_selector=*/nullptr);
7724 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7725
7726 // Reset encoder for new configuration to take effect.
7727 ConfigureEncoder(video_encoder_config_.Copy());
7728
7729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7730 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7731 /*round_trip_time_ms=*/0,
7732 /*cwnd_reduce_ratio=*/0);
7733 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7734
7735 ON_CALL(video_encoder, InitEncode(_, _))
7736 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7737
7738 rtc::Event encode_attempted;
7739 EXPECT_CALL(switch_callback,
7740 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7741 /*allow_default_fallback=*/true))
7742 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7743
7744 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007745 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007746
7747 AdvanceTime(TimeDelta::Zero());
7748
7749 video_stream_encoder_->Stop();
7750
7751 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007752 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007753 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7754 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007755 video_stream_encoder_.reset();
7756}
7757
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007758TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7759 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7760 // VideoEncoder is passed in encoder_factory, it checks whether
7761 // Codec Switch occurs without a crash.
7762 constexpr int kSufficientBitrateToNotDrop = 1000;
7763 constexpr int kDontCare = 100;
7764
7765 NiceMock<MockEncoderSelector> encoder_selector;
7766 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7767 video_send_config_.encoder_settings.encoder_switch_request_callback =
7768 &switch_callback;
7769 auto encoder_factory =
7770 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7771 /*encoder=*/nullptr, &encoder_selector);
7772 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7773
7774 // Reset encoder for new configuration to take effect.
7775 ConfigureEncoder(video_encoder_config_.Copy());
7776 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7777 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7778 // not fail.
7779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7780 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7781 /*stable_target_bitrate=*/
7782 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7783 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7784 /*fraction_lost=*/0,
7785 /*round_trip_time_ms=*/0,
7786 /*cwnd_reduce_ratio=*/0);
7787 ON_CALL(encoder_selector, OnEncoderBroken)
7788 .WillByDefault(Return(SdpVideoFormat("AV2")));
7789 rtc::Event encode_attempted;
7790 EXPECT_CALL(switch_callback,
7791 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7792 /*allow_default_fallback=*/_))
7793 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7794
7795 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007796 encode_attempted.Wait(TimeDelta::Seconds(3));
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007797
7798 AdvanceTime(TimeDelta::Zero());
7799
7800 video_stream_encoder_->Stop();
7801
7802 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7803 // to it's factory, so in order for the encoder instance in the
7804 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7805 // reset the `video_stream_encoder_` here.
7806 video_stream_encoder_.reset();
7807}
7808
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007809TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007810 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007811 const int kFrameWidth = 320;
7812 const int kFrameHeight = 180;
7813
7814 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007815 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007817 /*target_bitrate=*/rate,
7818 /*stable_target_bitrate=*/rate,
7819 /*link_allocation=*/rate,
7820 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007821 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007822 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007823
7824 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007825 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007826 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7827 frame.set_rotation(kVideoRotation_270);
7828 video_source_.IncomingCapturedFrame(frame);
7829 WaitForEncodedFrame(timestamp_ms);
7830 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7831
7832 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007833 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007835 /*target_bitrate=*/new_stable_rate,
7836 /*stable_target_bitrate=*/new_stable_rate,
7837 /*link_allocation=*/rate,
7838 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007839 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007840 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007841 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7842 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7843 video_stream_encoder_->Stop();
7844}
7845
7846TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007847 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007848 const int kFrameWidth = 320;
7849 const int kFrameHeight = 180;
7850
7851 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007852 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007854 /*target_bitrate=*/rate,
7855 /*stable_target_bitrate=*/rate,
7856 /*link_allocation=*/rate,
7857 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007858 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007859 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007860
7861 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007862 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007863 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7864 frame.set_rotation(kVideoRotation_270);
7865 video_source_.IncomingCapturedFrame(frame);
7866 WaitForEncodedFrame(timestamp_ms);
7867 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7868
7869 // Set a higher target rate without changing the link_allocation. Should not
7870 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007871 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007873 /*target_bitrate=*/rate,
7874 /*stable_target_bitrate=*/new_stable_rate,
7875 /*link_allocation=*/rate,
7876 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007877 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007878 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007879 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7880 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7881 video_stream_encoder_->Stop();
7882}
7883
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007884TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007885 test::ScopedKeyValueConfig field_trials(
7886 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007887 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7888 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7889 const int kFramerateFps = 30;
7890 const int kWidth = 1920;
7891 const int kHeight = 1080;
7892 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7893 // Works on screenshare mode.
7894 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7895 // We rely on the automatic resolution adaptation, but we handle framerate
7896 // adaptation manually by mocking the stats proxy.
7897 video_source_.set_adaptation_enabled(true);
7898
7899 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007901 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007902 video_stream_encoder_->SetSource(&video_source_,
7903 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007904 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007905
7906 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7907 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7908
7909 // Pass enough frames with the full update to trigger animation detection.
7910 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007911 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007912 frame.set_ntp_time_ms(timestamp_ms);
7913 frame.set_timestamp_us(timestamp_ms * 1000);
7914 video_source_.IncomingCapturedFrame(frame);
7915 WaitForEncodedFrame(timestamp_ms);
7916 }
7917
7918 // Resolution should be limited.
7919 rtc::VideoSinkWants expected;
7920 expected.max_framerate_fps = kFramerateFps;
7921 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007922 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007923
7924 // Pass one frame with no known update.
7925 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007926 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007927 frame.set_ntp_time_ms(timestamp_ms);
7928 frame.set_timestamp_us(timestamp_ms * 1000);
7929 frame.clear_update_rect();
7930
7931 video_source_.IncomingCapturedFrame(frame);
7932 WaitForEncodedFrame(timestamp_ms);
7933
7934 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007935 EXPECT_THAT(video_source_.sink_wants(),
7936 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007937
7938 video_stream_encoder_->Stop();
7939}
7940
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007941TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7942 const int kWidth = 720; // 540p adapted down.
7943 const int kHeight = 405;
7944 const int kNumFrames = 3;
7945 // Works on screenshare mode.
7946 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7947 /*num_spatial_layers=*/2, /*screenshare=*/true);
7948
7949 video_source_.set_adaptation_enabled(true);
7950
7951 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007952 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007953
7954 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7955
7956 // Pass enough frames with the full update to trigger animation detection.
7957 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007958 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007959 frame.set_ntp_time_ms(timestamp_ms);
7960 frame.set_timestamp_us(timestamp_ms * 1000);
7961 video_source_.IncomingCapturedFrame(frame);
7962 WaitForEncodedFrame(timestamp_ms);
7963 }
7964
7965 video_stream_encoder_->Stop();
7966}
7967
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007968TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7969 const float downscale_factors[] = {4.0, 2.0, 1.0};
7970 const int number_layers =
7971 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7972 VideoEncoderConfig config;
7973 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7974 for (int i = 0; i < number_layers; ++i) {
7975 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7976 config.simulcast_layers[i].active = true;
7977 }
7978 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007979 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007980 "VP8", /*max qp*/ 56, /*screencast*/ false,
7981 /*screenshare enabled*/ false);
7982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007983 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7984 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007985
7986 // First initialization.
7987 // Encoder should be initialized. Next frame should be key frame.
7988 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7989 sink_.SetNumExpectedLayers(number_layers);
7990 int64_t timestamp_ms = kFrameIntervalMs;
7991 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7992 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007993 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007994 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7995 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7996 VideoFrameType::kVideoFrameKey,
7997 VideoFrameType::kVideoFrameKey}));
7998
7999 // Disable top layer.
8000 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8001 config.simulcast_layers[number_layers - 1].active = false;
8002 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8003 sink_.SetNumExpectedLayers(number_layers - 1);
8004 timestamp_ms += kFrameIntervalMs;
8005 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8006 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008007 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008008 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8009 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8010 VideoFrameType::kVideoFrameDelta,
8011 VideoFrameType::kVideoFrameDelta}));
8012
8013 // Re-enable top layer.
8014 // Encoder should be re-initialized. Next frame should be key frame.
8015 config.simulcast_layers[number_layers - 1].active = true;
8016 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8017 sink_.SetNumExpectedLayers(number_layers);
8018 timestamp_ms += kFrameIntervalMs;
8019 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8020 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008021 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008022 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8023 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8024 VideoFrameType::kVideoFrameKey,
8025 VideoFrameType::kVideoFrameKey}));
8026
8027 // Top layer max rate change.
8028 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8029 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8030 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8031 sink_.SetNumExpectedLayers(number_layers);
8032 timestamp_ms += kFrameIntervalMs;
8033 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8034 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008035 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008036 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8037 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8038 VideoFrameType::kVideoFrameDelta,
8039 VideoFrameType::kVideoFrameDelta}));
8040
8041 // Top layer resolution change.
8042 // Encoder should be re-initialized. Next frame should be key frame.
8043 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8044 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8045 sink_.SetNumExpectedLayers(number_layers);
8046 timestamp_ms += kFrameIntervalMs;
8047 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8048 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008049 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008050 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8051 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8052 VideoFrameType::kVideoFrameKey,
8053 VideoFrameType::kVideoFrameKey}));
8054 video_stream_encoder_->Stop();
8055}
8056
Henrik Boström1124ed12021-02-25 10:30:39 +01008057TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8058 const int kFrameWidth = 1280;
8059 const int kFrameHeight = 720;
8060
8061 SetUp();
8062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008063 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008064
8065 // Capturing a frame should reconfigure the encoder and expose the encoder
8066 // resolution, which is the same as the input frame.
8067 int64_t timestamp_ms = kFrameIntervalMs;
8068 video_source_.IncomingCapturedFrame(
8069 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8070 WaitForEncodedFrame(timestamp_ms);
8071 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8072 EXPECT_THAT(video_source_.sink_wants().resolutions,
8073 ::testing::ElementsAreArray(
8074 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8075
8076 video_stream_encoder_->Stop();
8077}
8078
8079TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8080 // Pick downscale factors such that we never encode at full resolution - this
8081 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008082 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008083 // encoder should not ask for the frame resolution. This allows video frames
8084 // to have the appearence of one resolution but optimize its internal buffers
8085 // for what is actually encoded.
8086 const size_t kNumSimulcastLayers = 3u;
8087 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8088 const int kFrameWidth = 1280;
8089 const int kFrameHeight = 720;
8090 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8091 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8092 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8093 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8094 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8095 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8096
8097 VideoEncoderConfig config;
8098 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8099 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8100 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8101 config.simulcast_layers[i].active = true;
8102 }
8103 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008104 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008105 "VP8", /*max qp*/ 56, /*screencast*/ false,
8106 /*screenshare enabled*/ false);
8107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008108 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8109 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008110
8111 // Capture a frame with all layers active.
8112 int64_t timestamp_ms = kFrameIntervalMs;
8113 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8114 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8115 video_source_.IncomingCapturedFrame(
8116 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8117 WaitForEncodedFrame(timestamp_ms);
8118 // Expect encoded resolutions to match the expected simulcast layers.
8119 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8120 EXPECT_THAT(
8121 video_source_.sink_wants().resolutions,
8122 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8123
8124 // Capture a frame with one of the layers inactive.
8125 timestamp_ms += kFrameIntervalMs;
8126 config.simulcast_layers[2].active = false;
8127 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8128 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8129 video_source_.IncomingCapturedFrame(
8130 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8131 WaitForEncodedFrame(timestamp_ms);
8132
8133 // Expect encoded resolutions to match the expected simulcast layers.
8134 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8135 EXPECT_THAT(video_source_.sink_wants().resolutions,
8136 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8137
8138 // Capture a frame with all but one layer turned off.
8139 timestamp_ms += kFrameIntervalMs;
8140 config.simulcast_layers[1].active = false;
8141 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8142 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8143 video_source_.IncomingCapturedFrame(
8144 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8145 WaitForEncodedFrame(timestamp_ms);
8146
8147 // Expect encoded resolutions to match the expected simulcast layers.
8148 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8149 EXPECT_THAT(video_source_.sink_wants().resolutions,
8150 ::testing::ElementsAreArray({kLayer0Size}));
8151
8152 video_stream_encoder_->Stop();
8153}
8154
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008155TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008156 ResetEncoder("VP8", 1, 1, 1, false);
8157
Niels Möller8b692902021-06-14 12:04:57 +02008158 // Force encoder reconfig.
8159 video_source_.IncomingCapturedFrame(
8160 CreateFrame(1, codec_width_, codec_height_));
8161 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8162
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008163 // Set QP on encoded frame and pass the frame to encode complete callback.
8164 // Since QP is present QP parsing won't be triggered and the original value
8165 // should be kept.
8166 EncodedImage encoded_image;
8167 encoded_image.qp_ = 123;
8168 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8169 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8170 CodecSpecificInfo codec_info;
8171 codec_info.codecType = kVideoCodecVP8;
8172 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008173 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008174 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8175 video_stream_encoder_->Stop();
8176}
8177
8178TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008179 ResetEncoder("VP8", 1, 1, 1, false);
8180
Niels Möller8b692902021-06-14 12:04:57 +02008181 // Force encoder reconfig.
8182 video_source_.IncomingCapturedFrame(
8183 CreateFrame(1, codec_width_, codec_height_));
8184 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8185
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008186 // Pass an encoded frame without QP to encode complete callback. QP should be
8187 // parsed and set.
8188 EncodedImage encoded_image;
8189 encoded_image.qp_ = -1;
8190 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8191 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8192 CodecSpecificInfo codec_info;
8193 codec_info.codecType = kVideoCodecVP8;
8194 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008195 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008196 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8197 video_stream_encoder_->Stop();
8198}
8199
8200TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008201 webrtc::test::ScopedKeyValueConfig field_trials(
8202 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008203
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008204 ResetEncoder("VP8", 1, 1, 1, false);
8205
Niels Möller8b692902021-06-14 12:04:57 +02008206 // Force encoder reconfig.
8207 video_source_.IncomingCapturedFrame(
8208 CreateFrame(1, codec_width_, codec_height_));
8209 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8210
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008211 EncodedImage encoded_image;
8212 encoded_image.qp_ = -1;
8213 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8214 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8215 CodecSpecificInfo codec_info;
8216 codec_info.codecType = kVideoCodecVP8;
8217 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008218 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008219 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8220 video_stream_encoder_->Stop();
8221}
8222
Sergey Silkind19e3b92021-03-16 10:05:30 +00008223TEST_F(VideoStreamEncoderTest,
8224 QualityScalingNotAllowed_QualityScalingDisabled) {
8225 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8226
8227 // Disable scaling settings in encoder info.
8228 fake_encoder_.SetQualityScaling(false);
8229 // Disable quality scaling in encoder config.
8230 video_encoder_config.is_quality_scaling_allowed = false;
8231 ConfigureEncoder(std::move(video_encoder_config));
8232
8233 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008234 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008235
8236 test::FrameForwarder source;
8237 video_stream_encoder_->SetSource(
8238 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8239 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8240 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8241
8242 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8243 WaitForEncodedFrame(1);
8244 video_stream_encoder_->TriggerQualityLow();
8245 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8246
8247 video_stream_encoder_->Stop();
8248}
8249
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008250TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8251 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8252
8253 // Disable scaling settings in encoder info.
8254 fake_encoder_.SetQualityScaling(false);
8255 // Set QP trusted in encoder info.
8256 fake_encoder_.SetIsQpTrusted(true);
8257 // Enable quality scaling in encoder config.
8258 video_encoder_config.is_quality_scaling_allowed = false;
8259 ConfigureEncoder(std::move(video_encoder_config));
8260
8261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008262 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008263
8264 test::FrameForwarder source;
8265 video_stream_encoder_->SetSource(
8266 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8267 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8269
8270 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8271 WaitForEncodedFrame(1);
8272 video_stream_encoder_->TriggerQualityLow();
8273 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8274
8275 video_stream_encoder_->Stop();
8276}
8277
Shuhai Pengf2707702021-09-29 17:19:44 +08008278TEST_F(VideoStreamEncoderTest,
8279 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8280 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8281
8282 // Disable scaling settings in encoder info.
8283 fake_encoder_.SetQualityScaling(false);
8284 // Set QP trusted in encoder info.
8285 fake_encoder_.SetIsQpTrusted(true);
8286 // Enable quality scaling in encoder config.
8287 video_encoder_config.is_quality_scaling_allowed = false;
8288 ConfigureEncoder(std::move(video_encoder_config));
8289
8290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008291 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008292
8293 test::FrameForwarder source;
8294 video_stream_encoder_->SetSource(
8295 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8296 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8298
8299 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8300 WaitForEncodedFrame(1);
8301 video_stream_encoder_->TriggerQualityLow();
8302 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8303
8304 video_stream_encoder_->Stop();
8305}
8306
8307TEST_F(VideoStreamEncoderTest,
8308 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8309 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8310
8311 // Disable scaling settings in encoder info.
8312 fake_encoder_.SetQualityScaling(false);
8313 // Set QP trusted in encoder info.
8314 fake_encoder_.SetIsQpTrusted(false);
8315 // Enable quality scaling in encoder config.
8316 video_encoder_config.is_quality_scaling_allowed = false;
8317 ConfigureEncoder(std::move(video_encoder_config));
8318
8319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008320 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008321
8322 test::FrameForwarder source;
8323 video_stream_encoder_->SetSource(
8324 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8325 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8326 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8327
8328 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8329 WaitForEncodedFrame(1);
8330 video_stream_encoder_->TriggerQualityLow();
8331 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8332
8333 video_stream_encoder_->Stop();
8334}
8335
8336TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8337 // Set QP trusted in encoder info.
8338 fake_encoder_.SetIsQpTrusted(false);
8339
8340 const int MinEncBitrateKbps = 30;
8341 const int MaxEncBitrateKbps = 100;
8342 const int MinStartBitrateKbp = 50;
8343 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8344 /*frame_size_pixels=*/codec_width_ * codec_height_,
8345 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8346 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8347 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8348
8349 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008350 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008351
8352 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8353
8354 VideoEncoderConfig video_encoder_config;
8355 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8356 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8357 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8358 MinEncBitrateKbps * 1000;
8359 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8360 kMaxPayloadLength);
8361
8362 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8363 WaitForEncodedFrame(1);
8364 EXPECT_EQ(
8365 MaxEncBitrateKbps,
8366 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8367 EXPECT_EQ(
8368 MinEncBitrateKbps,
8369 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8370
8371 video_stream_encoder_->Stop();
8372}
8373
8374TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8375 // Set QP trusted in encoder info.
8376 fake_encoder_.SetIsQpTrusted(false);
8377
8378 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8379 EncoderInfoSettings::
8380 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8381 codec_width_ * codec_height_,
8382 EncoderInfoSettings::
8383 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8384 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8385
8386 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8387 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8388 const int TargetEncBitrate = MaxEncBitrate;
8389 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8390 DataRate::BitsPerSec(TargetEncBitrate),
8391 DataRate::BitsPerSec(TargetEncBitrate),
8392 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8393
8394 VideoEncoderConfig video_encoder_config;
8395 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8396 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8397 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8398 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8399 kMaxPayloadLength);
8400
8401 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8402 WaitForEncodedFrame(1);
8403 EXPECT_EQ(
8404 MaxEncBitrate / 1000,
8405 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8406 EXPECT_EQ(
8407 MinEncBitrate / 1000,
8408 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8409
8410 video_stream_encoder_->Stop();
8411}
8412
Erik Språnge4589cb2022-04-06 16:44:30 +02008413TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8414 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8415 /*num_spatial_layers=*/1,
8416 /*screenshare=*/false, /*allocation_callback_type=*/
8417 VideoStreamEncoder::BitrateAllocationCallbackType::
8418 kVideoBitrateAllocationWhenScreenSharing,
8419 /*num_cores=*/3);
8420
8421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8422 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8423 video_source_.IncomingCapturedFrame(
8424 CreateFrame(1, /*width=*/320, /*height=*/180));
8425 WaitForEncodedFrame(1);
8426 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8427 VideoCodecComplexity::kComplexityNormal);
8428 video_stream_encoder_->Stop();
8429}
8430
8431TEST_F(VideoStreamEncoderTest,
8432 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8433 webrtc::test::ScopedKeyValueConfig field_trials(
8434 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8435
8436 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8437 /*num_spatial_layers=*/1,
8438 /*screenshare=*/false, /*allocation_callback_type=*/
8439 VideoStreamEncoder::BitrateAllocationCallbackType::
8440 kVideoBitrateAllocationWhenScreenSharing,
8441 /*num_cores=*/2);
8442
8443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8444 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8445 video_source_.IncomingCapturedFrame(
8446 CreateFrame(1, /*width=*/320, /*height=*/180));
8447 WaitForEncodedFrame(1);
8448 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8449 VideoCodecComplexity::kComplexityNormal);
8450 video_stream_encoder_->Stop();
8451}
8452
8453TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8454 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8455 /*num_spatial_layers=*/1,
8456 /*screenshare=*/false, /*allocation_callback_type=*/
8457 VideoStreamEncoder::BitrateAllocationCallbackType::
8458 kVideoBitrateAllocationWhenScreenSharing,
8459 /*num_cores=*/2);
8460
8461 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8462 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8463 video_source_.IncomingCapturedFrame(
8464 CreateFrame(1, /*width=*/320, /*height=*/180));
8465 WaitForEncodedFrame(1);
8466 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8467 VideoCodecComplexity::kComplexityLow);
8468 video_stream_encoder_->Stop();
8469}
8470
Sergey Silkind19e3b92021-03-16 10:05:30 +00008471#if !defined(WEBRTC_IOS)
8472// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8473// disabled by default on iOS.
8474TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8475 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8476
8477 // Disable scaling settings in encoder info.
8478 fake_encoder_.SetQualityScaling(false);
8479 // Enable quality scaling in encoder config.
8480 video_encoder_config.is_quality_scaling_allowed = true;
8481 ConfigureEncoder(std::move(video_encoder_config));
8482
8483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008484 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008485
8486 test::FrameForwarder source;
8487 video_stream_encoder_->SetSource(
8488 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8489 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8490 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8491
8492 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8493 WaitForEncodedFrame(1);
8494 video_stream_encoder_->TriggerQualityLow();
8495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8496
8497 video_stream_encoder_->Stop();
8498}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008499
8500TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8501 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8502
8503 // Disable scaling settings in encoder info.
8504 fake_encoder_.SetQualityScaling(false);
8505 // Set QP trusted in encoder info.
8506 fake_encoder_.SetIsQpTrusted(true);
8507 // Enable quality scaling in encoder config.
8508 video_encoder_config.is_quality_scaling_allowed = true;
8509 ConfigureEncoder(std::move(video_encoder_config));
8510
8511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008512 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008513
8514 test::FrameForwarder source;
8515 video_stream_encoder_->SetSource(
8516 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8517 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8518 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8519
8520 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8521 WaitForEncodedFrame(1);
8522 video_stream_encoder_->TriggerQualityLow();
8523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8524
8525 video_stream_encoder_->Stop();
8526}
Shuhai Pengf2707702021-09-29 17:19:44 +08008527
8528TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8529 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8530
8531 // Disable scaling settings in encoder info.
8532 fake_encoder_.SetQualityScaling(false);
8533 // Set QP not trusted in encoder info.
8534 fake_encoder_.SetIsQpTrusted(false);
8535 // Enable quality scaling in encoder config.
8536 video_encoder_config.is_quality_scaling_allowed = true;
8537 ConfigureEncoder(std::move(video_encoder_config));
8538
8539 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008540 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008541
8542 test::FrameForwarder source;
8543 video_stream_encoder_->SetSource(
8544 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8545 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8546 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8547
8548 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8549 WaitForEncodedFrame(1);
8550 video_stream_encoder_->TriggerQualityLow();
8551 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8552 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8553 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8554
8555 video_stream_encoder_->Stop();
8556}
8557
8558TEST_F(VideoStreamEncoderTest,
8559 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8560 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8561
8562 // Disable scaling settings in encoder info.
8563 fake_encoder_.SetQualityScaling(false);
8564 // Set QP trusted in encoder info.
8565 fake_encoder_.SetIsQpTrusted(true);
8566 // Enable quality scaling in encoder config.
8567 video_encoder_config.is_quality_scaling_allowed = true;
8568 ConfigureEncoder(std::move(video_encoder_config));
8569
8570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008571 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008572
8573 test::FrameForwarder source;
8574 video_stream_encoder_->SetSource(
8575 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8576 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8577 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8578
8579 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8580 WaitForEncodedFrame(1);
8581 video_stream_encoder_->TriggerQualityLow();
8582 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8584
8585 video_stream_encoder_->Stop();
8586}
8587
8588TEST_F(VideoStreamEncoderTest,
8589 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8590 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8591
8592 // Disable scaling settings in encoder info.
8593 fake_encoder_.SetQualityScaling(false);
8594 // Set QP trusted in encoder info.
8595 fake_encoder_.SetIsQpTrusted(false);
8596 // Enable quality scaling in encoder config.
8597 video_encoder_config.is_quality_scaling_allowed = true;
8598 ConfigureEncoder(std::move(video_encoder_config));
8599
8600 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008601 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008602
8603 test::FrameForwarder source;
8604 video_stream_encoder_->SetSource(
8605 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8606 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8607 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8608
8609 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8610 WaitForEncodedFrame(1);
8611 video_stream_encoder_->TriggerQualityLow();
8612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8613
8614 video_stream_encoder_->Stop();
8615}
8616
Erik Språng5e13d052022-08-02 11:42:49 +02008617TEST_F(VideoStreamEncoderTest,
8618 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8619 // Send a native frame before encoder rates have been set. The encoder is
8620 // seen as paused at this time.
8621 rtc::Event frame_destroyed_event;
8622 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8623 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8624
8625 // Frame should be dropped and destroyed.
8626 ExpectDroppedFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00008627 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
Erik Språng5e13d052022-08-02 11:42:49 +02008628 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8629
8630 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8631 // frame.
8632 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8633 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8634 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8635 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8636
8637 video_stream_encoder_->Stop();
8638}
8639
Erik Språnge4589cb2022-04-06 16:44:30 +02008640#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008641
Henrik Boström56db9ff2021-03-24 09:06:45 +01008642// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8643class VideoStreamEncoderWithRealEncoderTest
8644 : public VideoStreamEncoderTest,
8645 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8646 public:
8647 VideoStreamEncoderWithRealEncoderTest()
8648 : VideoStreamEncoderTest(),
8649 codec_type_(std::get<0>(GetParam())),
8650 allow_i420_conversion_(std::get<1>(GetParam())) {}
8651
8652 void SetUp() override {
8653 VideoStreamEncoderTest::SetUp();
8654 std::unique_ptr<VideoEncoder> encoder;
8655 switch (codec_type_) {
8656 case kVideoCodecVP8:
8657 encoder = VP8Encoder::Create();
8658 break;
8659 case kVideoCodecVP9:
8660 encoder = VP9Encoder::Create();
8661 break;
8662 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008663 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008664 break;
8665 case kVideoCodecH264:
8666 encoder =
8667 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8668 break;
8669 case kVideoCodecMultiplex:
8670 mock_encoder_factory_for_multiplex_ =
8671 std::make_unique<MockVideoEncoderFactory>();
8672 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8673 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8674 .WillRepeatedly([] { return VP8Encoder::Create(); });
8675 encoder = std::make_unique<MultiplexEncoderAdapter>(
8676 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8677 false);
8678 break;
8679 default:
Artem Titovd3251962021-11-15 16:57:07 +01008680 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008681 }
8682 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8683 }
8684
8685 void TearDown() override {
8686 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008687 // Ensure `video_stream_encoder_` is destroyed before
8688 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008689 video_stream_encoder_.reset();
8690 VideoStreamEncoderTest::TearDown();
8691 }
8692
8693 protected:
8694 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8695 std::unique_ptr<VideoEncoder> encoder) {
8696 // Configure VSE to use the encoder.
8697 encoder_ = std::move(encoder);
8698 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8699 encoder_.get(), &encoder_selector_);
8700 video_send_config_.encoder_settings.encoder_factory =
8701 encoder_proxy_factory_.get();
8702 VideoEncoderConfig video_encoder_config;
8703 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8704 video_encoder_config_ = video_encoder_config.Copy();
8705 ConfigureEncoder(video_encoder_config_.Copy());
8706
8707 // Set bitrate to ensure frame is not dropped.
8708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008709 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008710 }
8711
8712 const VideoCodecType codec_type_;
8713 const bool allow_i420_conversion_;
8714 NiceMock<MockEncoderSelector> encoder_selector_;
8715 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8716 std::unique_ptr<VideoEncoder> encoder_;
8717 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8718};
8719
8720TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8721 auto native_i420_frame = test::CreateMappableNativeFrame(
8722 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8723 video_source_.IncomingCapturedFrame(native_i420_frame);
8724 WaitForEncodedFrame(codec_width_, codec_height_);
8725
8726 auto mappable_native_buffer =
8727 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8728 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8729 mappable_native_buffer->GetMappedFramedBuffers();
8730 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8731 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8732 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8733 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8734}
8735
8736TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8737 auto native_nv12_frame = test::CreateMappableNativeFrame(
8738 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8739 video_source_.IncomingCapturedFrame(native_nv12_frame);
8740 WaitForEncodedFrame(codec_width_, codec_height_);
8741
8742 auto mappable_native_buffer =
8743 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8744 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8745 mappable_native_buffer->GetMappedFramedBuffers();
8746 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8747 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8748 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8749 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8750
8751 if (!allow_i420_conversion_) {
8752 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8753 }
8754}
8755
Erik Språng7444b192021-06-02 14:02:13 +02008756TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8757 if (codec_type_ == kVideoCodecMultiplex) {
8758 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8759 return;
8760 }
8761
8762 const size_t kNumSpatialLayers = 3u;
8763 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8764 const int kFrameWidth = 1280;
8765 const int kFrameHeight = 720;
8766 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8767 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8768 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8769 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8770 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8771 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8772
8773 VideoEncoderConfig config;
8774 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8775 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008776 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008777 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8778 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8779 vp9_settings.numberOfTemporalLayers = 3;
8780 vp9_settings.automaticResizeOn = false;
8781 config.encoder_specific_settings =
8782 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8783 vp9_settings);
8784 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8785 /*fps=*/30.0,
8786 /*first_active_layer=*/0,
8787 /*num_spatial_layers=*/3,
8788 /*num_temporal_layers=*/3,
8789 /*is_screenshare=*/false);
8790 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8791 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008792 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008793 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8794 /*fps=*/30.0,
8795 /*first_active_layer=*/0,
8796 /*num_spatial_layers=*/3,
8797 /*num_temporal_layers=*/3,
8798 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008799 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008800 } else {
8801 // Simulcast for VP8/H264.
8802 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8803 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8804 config.simulcast_layers[i].scale_resolution_down_by =
8805 kDownscaleFactors[i];
8806 config.simulcast_layers[i].active = true;
8807 }
8808 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8809 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008810 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008811 }
8812 }
8813
8814 auto set_layer_active = [&](int layer_idx, bool active) {
8815 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8816 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8817 config.spatial_layers[layer_idx].active = active;
8818 } else {
8819 config.simulcast_layers[layer_idx].active = active;
8820 }
8821 };
8822
8823 config.video_stream_factory =
8824 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8825 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8826 /*screencast*/ false,
8827 /*screenshare enabled*/ false);
8828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008829 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8830 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008831
8832 // Capture a frame with all layers active.
8833 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8834 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8835 int64_t timestamp_ms = kFrameIntervalMs;
8836 video_source_.IncomingCapturedFrame(
8837 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8838
8839 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8841
8842 // Capture a frame with one of the layers inactive.
8843 set_layer_active(2, false);
8844 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8845 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8846 timestamp_ms += kFrameIntervalMs;
8847 video_source_.IncomingCapturedFrame(
8848 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8849 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8850
8851 // New target bitrates signaled based on lower resolution.
8852 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8854 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8855 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8856
8857 // Re-enable the top layer.
8858 set_layer_active(2, true);
8859 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8860 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8861 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8862
8863 // Bitrate target adjusted back up to enable HD layer...
8864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8865 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8866 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8868
8869 // ...then add a new frame.
8870 timestamp_ms += kFrameIntervalMs;
8871 video_source_.IncomingCapturedFrame(
8872 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8873 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8874 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8875
8876 video_stream_encoder_->Stop();
8877}
8878
Henrik Boström56db9ff2021-03-24 09:06:45 +01008879std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8880 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8881 VideoCodecType codec_type = std::get<0>(info.param);
8882 bool allow_i420_conversion = std::get<1>(info.param);
8883 std::string str;
8884 switch (codec_type) {
8885 case kVideoCodecGeneric:
8886 str = "Generic";
8887 break;
8888 case kVideoCodecVP8:
8889 str = "VP8";
8890 break;
8891 case kVideoCodecVP9:
8892 str = "VP9";
8893 break;
8894 case kVideoCodecAV1:
8895 str = "AV1";
8896 break;
8897 case kVideoCodecH264:
8898 str = "H264";
8899 break;
8900 case kVideoCodecMultiplex:
8901 str = "Multiplex";
8902 break;
8903 default:
Artem Titovd3251962021-11-15 16:57:07 +01008904 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008905 }
8906 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8907 return str;
8908}
8909
8910constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8911 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8912constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8913 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8914constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008915 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008916constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8917 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8918#if defined(WEBRTC_USE_H264)
8919constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8920 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8921
8922// The windows compiler does not tolerate #if statements inside the
8923// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8924// and without H264).
8925INSTANTIATE_TEST_SUITE_P(
8926 All,
8927 VideoStreamEncoderWithRealEncoderTest,
8928 ::testing::Values(kVP8DisallowConversion,
8929 kVP9DisallowConversion,
8930 kAV1AllowConversion,
8931 kMultiplexDisallowConversion,
8932 kH264AllowConversion),
8933 TestParametersVideoCodecAndAllowI420ConversionToString);
8934#else
8935INSTANTIATE_TEST_SUITE_P(
8936 All,
8937 VideoStreamEncoderWithRealEncoderTest,
8938 ::testing::Values(kVP8DisallowConversion,
8939 kVP9DisallowConversion,
8940 kAV1AllowConversion,
8941 kMultiplexDisallowConversion),
8942 TestParametersVideoCodecAndAllowI420ConversionToString);
8943#endif
8944
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008945class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8946 protected:
8947 void RunTest(const std::vector<VideoStream>& configs,
8948 const int expected_num_init_encode) {
8949 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008950 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008951 InsertFrameAndWaitForEncoded();
8952 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8953 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008954 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8955 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008956
8957 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8958 ConfigureEncoder(configs[1]);
8959 InsertFrameAndWaitForEncoded();
8960 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8961 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008962 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008963 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008964 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008965
8966 video_stream_encoder_->Stop();
8967 }
8968
8969 void ConfigureEncoder(const VideoStream& stream) {
8970 VideoEncoderConfig config;
8971 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8972 config.max_bitrate_bps = stream.max_bitrate_bps;
8973 config.simulcast_layers[0] = stream;
8974 config.video_stream_factory =
8975 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8976 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8977 /*conference_mode=*/false);
8978 video_stream_encoder_->ConfigureEncoder(std::move(config),
8979 kMaxPayloadLength);
8980 }
8981
8982 void OnBitrateUpdated(DataRate bitrate) {
8983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8984 bitrate, bitrate, bitrate, 0, 0, 0);
8985 }
8986
8987 void InsertFrameAndWaitForEncoded() {
8988 timestamp_ms_ += kFrameIntervalMs;
8989 video_source_.IncomingCapturedFrame(
8990 CreateFrame(timestamp_ms_, kWidth, kHeight));
8991 sink_.WaitForEncodedFrame(timestamp_ms_);
8992 }
8993
8994 void ExpectEqual(const VideoCodec& actual,
8995 const VideoStream& expected) const {
8996 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8997 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8998 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8999 static_cast<unsigned int>(expected.min_bitrate_bps));
9000 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9001 static_cast<unsigned int>(expected.max_bitrate_bps));
9002 EXPECT_EQ(actual.simulcastStream[0].width,
9003 kWidth / expected.scale_resolution_down_by);
9004 EXPECT_EQ(actual.simulcastStream[0].height,
9005 kHeight / expected.scale_resolution_down_by);
9006 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9007 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009008 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009009 }
9010
9011 VideoStream DefaultConfig() const {
9012 VideoStream stream;
9013 stream.max_framerate = 25;
9014 stream.min_bitrate_bps = 35000;
9015 stream.max_bitrate_bps = 900000;
9016 stream.scale_resolution_down_by = 1.0;
9017 stream.num_temporal_layers = 1;
9018 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009019 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009020 return stream;
9021 }
9022
9023 const int kWidth = 640;
9024 const int kHeight = 360;
9025 int64_t timestamp_ms_ = 0;
9026};
9027
9028TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9029 VideoStream config1 = DefaultConfig();
9030 VideoStream config2 = config1;
9031 config2.max_framerate++;
9032
9033 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9034}
9035
9036TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9037 VideoStream config1 = DefaultConfig();
9038 VideoStream config2 = config1;
9039 config2.min_bitrate_bps += 10000;
9040
9041 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9042}
9043
9044TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9045 VideoStream config1 = DefaultConfig();
9046 VideoStream config2 = config1;
9047 config2.max_bitrate_bps += 100000;
9048
9049 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9050}
9051
9052TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9053 VideoStream config1 = DefaultConfig();
9054 VideoStream config2 = config1;
9055 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9056
9057 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9058}
9059
9060TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9061 VideoStream config1 = DefaultConfig();
9062 VideoStream config2 = config1;
9063 config2.scale_resolution_down_by *= 2;
9064
9065 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9066}
9067
9068TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9069 VideoStream config1 = DefaultConfig();
9070 VideoStream config2 = config1;
9071 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9072
9073 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9074}
9075
9076TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9077 VideoStream config1 = DefaultConfig();
9078 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009079 config2.scalability_mode = ScalabilityMode::kL2T1;
9080
9081 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9082}
9083
9084TEST_F(ReconfigureEncoderTest,
9085 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9086 VideoStream config1 = DefaultConfig();
9087 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009088 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009089 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009090
9091 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9092}
9093
Tommi62b01db2022-01-25 23:41:22 +01009094// Simple test that just creates and then immediately destroys an encoder.
9095// The purpose of the test is to make sure that nothing bad happens if the
9096// initialization step on the encoder queue, doesn't run.
9097TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9098 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9099 public:
9100 SuperLazyTaskQueue() = default;
9101 ~SuperLazyTaskQueue() override = default;
9102
9103 private:
9104 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009105 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009106 // meh.
9107 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009108 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9109 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009110 ASSERT_TRUE(false);
9111 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009112 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9113 TimeDelta delay) override {
9114 ADD_FAILURE();
9115 }
Tommi62b01db2022-01-25 23:41:22 +01009116 };
9117
9118 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009119 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009120 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009121 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9122 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009123 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009124 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9125 time_controller.GetClock());
9126 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9127 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9128 CreateBuiltinVideoBitrateAllocatorFactory();
9129 VideoStreamEncoderSettings encoder_settings{
9130 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9131 encoder_settings.encoder_factory = &encoder_factory;
9132 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9133
9134 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9135 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9136
9137 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9138 encoder_queue(new SuperLazyTaskQueue());
9139
9140 // Construct a VideoStreamEncoder instance and let it go out of scope without
9141 // doing anything else (including calling Stop()). This should be fine since
9142 // the posted init task will simply be deleted.
9143 auto encoder = std::make_unique<VideoStreamEncoder>(
9144 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009145 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9146 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009147 std::move(adapter), std::move(encoder_queue),
9148 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009149 kVideoBitrateAllocation,
9150 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009151
9152 // Stop the encoder explicitly. This additional step tests if we could
9153 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9154 // any more tasks.
9155 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009156}
9157
Markus Handellb4e96d42021-11-05 12:00:55 +01009158TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9159 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9160 auto* adapter_ptr = adapter.get();
9161 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009162 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9163 nullptr;
9164 EXPECT_CALL(*adapter_ptr, Initialize)
9165 .WillOnce(Invoke([&video_stream_encoder_callback](
9166 FrameCadenceAdapterInterface::Callback* callback) {
9167 video_stream_encoder_callback = callback;
9168 }));
9169 TaskQueueBase* encoder_queue = nullptr;
9170 auto video_stream_encoder =
9171 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009172
Markus Handelle59fee82021-12-23 09:29:23 +01009173 // First a call before we know the frame size and hence cannot compute the
9174 // number of simulcast layers.
9175 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9176 &FrameCadenceAdapterInterface::
9177 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009178 Eq(0u)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009179 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009180 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009181 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9182 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009183 factory.DepleteTaskQueues();
9184
9185 // Then a call as we've computed the number of simulcast layers after a passed
9186 // frame.
9187 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9188 &FrameCadenceAdapterInterface::
9189 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009190 Gt(0u)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009191 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009192 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009193 Mock::VerifyAndClearExpectations(adapter_ptr);
9194
Markus Handelle59fee82021-12-23 09:29:23 +01009195 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009196 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009197 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009198 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009199 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9200 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009201 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009202 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009203}
9204
9205TEST(VideoStreamEncoderFrameCadenceTest,
9206 ForwardsFramesIntoFrameCadenceAdapter) {
9207 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9208 auto* adapter_ptr = adapter.get();
9209 test::FrameForwarder video_source;
9210 SimpleVideoStreamEncoderFactory factory;
9211 auto video_stream_encoder = factory.Create(std::move(adapter));
9212 video_stream_encoder->SetSource(
9213 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9214
9215 EXPECT_CALL(*adapter_ptr, OnFrame);
9216 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9217 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009218 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009219}
9220
Markus Handellee225432021-11-29 12:35:12 +01009221TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9222 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9223 auto* adapter_ptr = adapter.get();
9224 test::FrameForwarder video_source;
9225 SimpleVideoStreamEncoderFactory factory;
9226 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9227 nullptr;
9228 EXPECT_CALL(*adapter_ptr, Initialize)
9229 .WillOnce(Invoke([&video_stream_encoder_callback](
9230 FrameCadenceAdapterInterface::Callback* callback) {
9231 video_stream_encoder_callback = callback;
9232 }));
9233 TaskQueueBase* encoder_queue = nullptr;
9234 auto video_stream_encoder =
9235 factory.Create(std::move(adapter), &encoder_queue);
9236
9237 // This is just to make the VSE operational. We'll feed a frame directly by
9238 // the callback interface.
9239 video_stream_encoder->SetSource(
9240 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9241
9242 VideoEncoderConfig video_encoder_config;
9243 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9244 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9245 /*max_data_payload_length=*/1000);
9246
9247 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9248 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009249 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009250 factory.DepleteTaskQueues();
9251}
9252
Markus Handell8d87c462021-12-16 11:37:16 +01009253TEST(VideoStreamEncoderFrameCadenceTest,
9254 DeactivatesActivatesLayersOnBitrateChanges) {
9255 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9256 auto* adapter_ptr = adapter.get();
9257 SimpleVideoStreamEncoderFactory factory;
9258 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9259 nullptr;
9260 EXPECT_CALL(*adapter_ptr, Initialize)
9261 .WillOnce(Invoke([&video_stream_encoder_callback](
9262 FrameCadenceAdapterInterface::Callback* callback) {
9263 video_stream_encoder_callback = callback;
9264 }));
9265 TaskQueueBase* encoder_queue = nullptr;
9266 auto video_stream_encoder =
9267 factory.Create(std::move(adapter), &encoder_queue);
9268
9269 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9270 // {150000, 450000}.
9271 VideoEncoderConfig video_encoder_config;
9272 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9273 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9274 kMaxPayloadLength);
9275 // Ensure an encoder is created.
9276 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9277
9278 // Both layers enabled at 1 MBit/s.
9279 video_stream_encoder->OnBitrateUpdated(
9280 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9281 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9282 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9283 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9284 factory.DepleteTaskQueues();
9285 Mock::VerifyAndClearExpectations(adapter_ptr);
9286
9287 // Layer 1 disabled at 200 KBit/s.
9288 video_stream_encoder->OnBitrateUpdated(
9289 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9290 DataRate::KilobitsPerSec(200), 0, 0, 0);
9291 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9292 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9293 factory.DepleteTaskQueues();
9294 Mock::VerifyAndClearExpectations(adapter_ptr);
9295
9296 // All layers off at suspended video.
9297 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9298 DataRate::Zero(), 0, 0, 0);
9299 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9300 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9301 factory.DepleteTaskQueues();
9302 Mock::VerifyAndClearExpectations(adapter_ptr);
9303
9304 // Both layers enabled again back at 1 MBit/s.
9305 video_stream_encoder->OnBitrateUpdated(
9306 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9307 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9308 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9309 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9310 factory.DepleteTaskQueues();
9311}
9312
9313TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9314 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9315 auto* adapter_ptr = adapter.get();
9316 SimpleVideoStreamEncoderFactory factory;
9317 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9318 nullptr;
9319 EXPECT_CALL(*adapter_ptr, Initialize)
9320 .WillOnce(Invoke([&video_stream_encoder_callback](
9321 FrameCadenceAdapterInterface::Callback* callback) {
9322 video_stream_encoder_callback = callback;
9323 }));
9324 TaskQueueBase* encoder_queue = nullptr;
9325 auto video_stream_encoder =
9326 factory.Create(std::move(adapter), &encoder_queue);
9327
9328 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9329 VideoEncoderConfig video_encoder_config;
9330 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9331 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9332 kMaxPayloadLength);
9333 video_stream_encoder->OnBitrateUpdated(
9334 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9335 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9336
9337 // Pass a frame which has unconverged results.
9338 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9339 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9340 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9341 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009342 encoded_image.qp_ = kVp8SteadyStateQpThreshold + 1;
Markus Handell8d87c462021-12-16 11:37:16 +01009343 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009344 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009345 return codec_specific;
9346 }));
9347 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9348 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9349 factory.DepleteTaskQueues();
9350 Mock::VerifyAndClearExpectations(adapter_ptr);
9351 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9352
9353 // Pass a frame which converges in layer 0 and not in layer 1.
9354 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9355 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9356 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9357 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009358 // This sets spatial index 0 content to be at target quality, while
9359 // index 1 content is not.
9360 encoded_image.qp_ = kVp8SteadyStateQpThreshold +
9361 (encoded_image.SpatialIndex() == 0 ? 0 : 1);
Markus Handell8d87c462021-12-16 11:37:16 +01009362 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009363 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009364 return codec_specific;
9365 }));
9366 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9367 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9368 factory.DepleteTaskQueues();
9369 Mock::VerifyAndClearExpectations(adapter_ptr);
9370 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9371}
9372
Markus Handell2e0f4f02021-12-21 19:14:58 +01009373TEST(VideoStreamEncoderFrameCadenceTest,
9374 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9375 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9376 auto* adapter_ptr = adapter.get();
9377 MockVideoSourceInterface mock_source;
9378 SimpleVideoStreamEncoderFactory factory;
9379 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9380 nullptr;
9381 EXPECT_CALL(*adapter_ptr, Initialize)
9382 .WillOnce(Invoke([&video_stream_encoder_callback](
9383 FrameCadenceAdapterInterface::Callback* callback) {
9384 video_stream_encoder_callback = callback;
9385 }));
9386 TaskQueueBase* encoder_queue = nullptr;
9387 auto video_stream_encoder =
9388 factory.Create(std::move(adapter), &encoder_queue);
9389 video_stream_encoder->SetSource(
9390 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9391 VideoEncoderConfig config;
9392 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9393 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9394 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9395 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9396 // Ensure the encoder is set up.
9397 factory.DepleteTaskQueues();
9398
Markus Handell818e7fb2021-12-30 13:01:33 +01009399 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9400 .WillOnce(Invoke([video_stream_encoder_callback] {
9401 video_stream_encoder_callback->RequestRefreshFrame();
9402 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009403 EXPECT_CALL(mock_source, RequestRefreshFrame);
9404 video_stream_encoder->SendKeyFrame();
9405 factory.DepleteTaskQueues();
9406 Mock::VerifyAndClearExpectations(adapter_ptr);
9407 Mock::VerifyAndClearExpectations(&mock_source);
9408
Markus Handell818e7fb2021-12-30 13:01:33 +01009409 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009410 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9411 video_stream_encoder->SendKeyFrame();
9412 factory.DepleteTaskQueues();
9413}
9414
Markus Handell818e7fb2021-12-30 13:01:33 +01009415TEST(VideoStreamEncoderFrameCadenceTest,
9416 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9417 SimpleVideoStreamEncoderFactory factory;
9418 auto encoder_queue =
9419 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9420 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9421
9422 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009423 test::ScopedKeyValueConfig field_trials(
9424 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009425 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009426 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9427 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009428 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9429
9430 MockVideoSourceInterface mock_source;
9431 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009432 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009433
9434 video_stream_encoder->SetSource(
9435 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9436 VideoEncoderConfig config;
9437 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9438 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9439 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9440
9441 // Eventually expect a refresh frame request when requesting a key frame
9442 // before initializing zero-hertz mode. This can happen in reality because the
9443 // threads invoking key frame requests and constraints setup aren't
9444 // synchronized.
9445 EXPECT_CALL(mock_source, RequestRefreshFrame);
9446 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009447 constexpr int kMaxFps = 30;
9448 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9449 factory.GetTimeController()->AdvanceTime(
9450 TimeDelta::Seconds(1) *
9451 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9452 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009453}
9454
perkj26091b12016-09-01 01:17:40 -07009455} // namespace webrtc