blob: cf3b2073c975ab709b7a110ff61ffa683b9be49a [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020055#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010056#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020057#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080059#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020060#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020061#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/encoder_settings.h"
63#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010064#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "test/gmock.h"
66#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010067#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010068#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020069#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090070#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020071#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010072#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020073#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070074
75namespace webrtc {
76
sprang57c2fff2017-01-16 06:24:02 -080077using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Ge;
82using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010083using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020084using ::testing::Le;
85using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010091using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020092using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080093
perkj803d97f2016-11-01 11:45:46 -070094namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010096const int kQpLow = 1;
97const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int kMinFramerateFps = 2;
99const int kMinBalancedFramerateFps = 7;
100const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -0800101const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200102const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
103const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
104const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
105const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800106const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700107const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200108const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200109const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800114
Asa Persson606d3cb2021-10-04 10:07:11 +0200115uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200116 0x00, 0x00, 0x03, 0x03, 0xF4,
117 0x05, 0x03, 0xC7, 0xE0, 0x1B,
118 0x41, 0x10, 0x8D, 0x00};
119
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100120const uint8_t kCodedFrameVp8Qp25[] = {
121 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
122 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
123 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
124
Markus Handell818e7fb2021-12-30 13:01:33 +0100125VideoFrame CreateSimpleNV12Frame() {
126 return VideoFrame::Builder()
127 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
128 /*width=*/16, /*height=*/16))
129 .build();
130}
131
Markus Handell8d87c462021-12-16 11:37:16 +0100132void PassAFrame(
133 TaskQueueBase* encoder_queue,
134 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
135 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200136 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
137 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
138 CreateSimpleNV12Frame());
139 });
Markus Handell8d87c462021-12-16 11:37:16 +0100140}
141
perkj803d97f2016-11-01 11:45:46 -0700142class TestBuffer : public webrtc::I420Buffer {
143 public:
144 TestBuffer(rtc::Event* event, int width, int height)
145 : I420Buffer(width, height), event_(event) {}
146
147 private:
148 friend class rtc::RefCountedObject<TestBuffer>;
149 ~TestBuffer() override {
150 if (event_)
151 event_->Set();
152 }
153 rtc::Event* const event_;
154};
155
Henrik Boström56db9ff2021-03-24 09:06:45 +0100156// A fake native buffer that can't be converted to I420. Upon scaling, it
157// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700158class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
159 public:
160 FakeNativeBuffer(rtc::Event* event, int width, int height)
161 : event_(event), width_(width), height_(height) {}
162 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
163 int width() const override { return width_; }
164 int height() const override { return height_; }
165 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
166 return nullptr;
167 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100168 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
169 int offset_x,
170 int offset_y,
171 int crop_width,
172 int crop_height,
173 int scaled_width,
174 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200175 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
176 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100177 }
Noah Richards51db4212019-06-12 06:59:12 -0700178
179 private:
180 friend class rtc::RefCountedObject<FakeNativeBuffer>;
181 ~FakeNativeBuffer() override {
182 if (event_)
183 event_->Set();
184 }
185 rtc::Event* const event_;
186 const int width_;
187 const int height_;
188};
189
Evan Shrubsole895556e2020-10-05 09:15:13 +0200190// A fake native buffer that is backed by an NV12 buffer.
191class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
192 public:
193 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
194 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
195
196 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
197 int width() const override { return nv12_buffer_->width(); }
198 int height() const override { return nv12_buffer_->height(); }
199 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
200 return nv12_buffer_->ToI420();
201 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200202 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
203 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
204 if (absl::c_find(types, Type::kNV12) != types.end()) {
205 return nv12_buffer_;
206 }
207 return nullptr;
208 }
Niels Möllerba2de582022-04-20 16:46:26 +0200209 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200210
211 private:
212 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
213 ~FakeNV12NativeBuffer() override {
214 if (event_)
215 event_->Set();
216 }
217 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
218 rtc::Event* const event_;
219};
220
Niels Möller7dc26b72017-12-06 10:27:48 +0100221class CpuOveruseDetectorProxy : public OveruseFrameDetector {
222 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200223 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
224 const FieldTrialsView& field_trials)
225 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200226 last_target_framerate_fps_(-1),
227 framerate_updated_event_(true /* manual_reset */,
228 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 virtual ~CpuOveruseDetectorProxy() {}
230
231 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 last_target_framerate_fps_ = framerate_fps;
234 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200235 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 }
237
238 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200239 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100240 return last_target_framerate_fps_;
241 }
242
Niels Möller4db138e2018-04-19 09:04:13 +0200243 CpuOveruseOptions GetOptions() { return options_; }
244
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
246
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 private:
Markus Handella3765182020-07-08 13:13:32 +0200248 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200250 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251};
252
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253class FakeVideoSourceRestrictionsListener
254 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200257 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200259 RTC_DCHECK(was_restrictions_updated_);
260 }
261
262 rtc::Event* restrictions_updated_event() {
263 return &restrictions_updated_event_;
264 }
265
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200266 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200267 void OnVideoSourceRestrictionsUpdated(
268 VideoSourceRestrictions restrictions,
269 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200270 rtc::scoped_refptr<Resource> reason,
271 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200272 was_restrictions_updated_ = true;
273 restrictions_updated_event_.Set();
274 }
275
276 private:
277 bool was_restrictions_updated_;
278 rtc::Event restrictions_updated_event_;
279};
280
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200281auto WantsFps(Matcher<int> fps_matcher) {
282 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
283 fps_matcher);
284}
285
286auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
287 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
288 AllOf(max_pixel_matcher, Gt(0)));
289}
290
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200291auto ResolutionMax() {
292 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200293 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200294 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
295 Eq(absl::nullopt)));
296}
297
298auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200311 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200312}
313
314auto FpsMaxResolutionMax() {
315 return AllOf(FpsMax(), ResolutionMax());
316}
317
318auto UnlimitedSinkWants() {
319 return AllOf(FpsUnlimited(), ResolutionMax());
320}
321
322auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
323 Matcher<int> fps_range_matcher;
324
325 if (last_frame_pixels <= 320 * 240) {
326 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200327 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200328 fps_range_matcher = AllOf(Ge(10), Le(15));
329 } else if (last_frame_pixels <= 640 * 480) {
330 fps_range_matcher = Ge(15);
331 } else {
332 fps_range_matcher = Eq(kDefaultFramerate);
333 }
334 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
335 fps_range_matcher);
336}
337
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200338auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
345}
346
347auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
349}
350
351auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
364}
365
366auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
367 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
368 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
369}
370
mflodmancc3d4422017-08-03 08:27:51 -0700371class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700372 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100373 VideoStreamEncoderUnderTest(
374 TimeController* time_controller,
375 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
376 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
377 encoder_queue,
378 SendStatisticsProxy* stats_proxy,
379 const VideoStreamEncoderSettings& settings,
380 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100381 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200382 const FieldTrialsView& field_trials,
383 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200384 : VideoStreamEncoder(
385 time_controller->GetClock(),
386 num_cores,
387 stats_proxy,
388 settings,
389 std::unique_ptr<OveruseFrameDetector>(
390 overuse_detector_proxy_ =
391 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
392 std::move(cadence_adapter),
393 std::move(encoder_queue),
394 allocation_callback_type,
395 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200396 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200397 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200398 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200399 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200400 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200401 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100404 }
perkj803d97f2016-11-01 11:45:46 -0700405
Henrik Boström381d1092020-05-12 18:49:07 +0200406 void SetSourceAndWaitForRestrictionsUpdated(
407 rtc::VideoSourceInterface<VideoFrame>* source,
408 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200409 FakeVideoSourceRestrictionsListener listener;
410 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 SetSource(source, degradation_preference);
412 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200414 }
415
416 void SetSourceAndWaitForFramerateUpdated(
417 rtc::VideoSourceInterface<VideoFrame>* source,
418 const DegradationPreference& degradation_preference) {
419 overuse_detector_proxy_->framerate_updated_event()->Reset();
420 SetSource(source, degradation_preference);
421 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
422 }
423
424 void OnBitrateUpdatedAndWaitForManagedResources(
425 DataRate target_bitrate,
426 DataRate stable_target_bitrate,
427 DataRate link_allocation,
428 uint8_t fraction_lost,
429 int64_t round_trip_time_ms,
430 double cwnd_reduce_ratio) {
431 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
432 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
433 // Bitrate is updated on the encoder queue.
434 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200435 }
436
kthelgason2fc52542017-03-03 00:24:41 -0800437 // This is used as a synchronisation mechanism, to make sure that the
438 // encoder queue is not blocked before we start sending it frames.
439 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100440 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800441 }
442
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200444 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200446 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200447 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 event.Set();
449 });
450 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100451 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200453
Henrik Boström91aa7322020-04-28 12:24:33 +0200454 void TriggerCpuUnderuse() {
455 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200456 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200457 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 event.Set();
459 });
460 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100461 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 }
kthelgason876222f2016-11-29 01:44:11 -0800463
Henrik Boström91aa7322020-04-28 12:24:33 +0200464 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200467 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200468 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 event.Set();
470 });
471 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100472 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200473 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200474 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200476 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200477 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 event.Set();
479 });
480 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100481 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200482 }
483
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200484 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100485 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200486 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
487 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200488 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700489};
490
Noah Richards51db4212019-06-12 06:59:12 -0700491// Simulates simulcast behavior and makes highest stream resolutions divisible
492// by 4.
493class CroppingVideoStreamFactory
494 : public VideoEncoderConfig::VideoStreamFactoryInterface {
495 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200496 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700497
498 private:
499 std::vector<VideoStream> CreateEncoderStreams(
500 int width,
501 int height,
502 const VideoEncoderConfig& encoder_config) override {
503 std::vector<VideoStream> streams = test::CreateVideoStreams(
504 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700505 return streams;
506 }
Noah Richards51db4212019-06-12 06:59:12 -0700507};
508
sprangb1ca0732017-02-01 08:38:12 -0800509class AdaptingFrameForwarder : public test::FrameForwarder {
510 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 explicit AdaptingFrameForwarder(TimeController* time_controller)
512 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700513 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800514
515 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200516 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800517 adaptation_enabled_ = enabled;
518 }
519
asaperssonfab67072017-04-04 05:51:49 -0700520 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800522 return adaptation_enabled_;
523 }
524
Henrik Boström1124ed12021-02-25 10:30:39 +0100525 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
526 // the resolution or frame rate was different than it is currently. If
527 // something else is modified, such as encoder resolutions, but the resolution
528 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700529 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200530 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700531 return last_wants_;
532 }
533
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200534 absl::optional<int> last_sent_width() const { return last_width_; }
535 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800536
sprangb1ca0732017-02-01 08:38:12 -0800537 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200538 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100539 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200540
sprangb1ca0732017-02-01 08:38:12 -0800541 int cropped_width = 0;
542 int cropped_height = 0;
543 int out_width = 0;
544 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700545 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000546 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
547 << "w=" << video_frame.width()
548 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700549 if (adapter_.AdaptFrameResolution(
550 video_frame.width(), video_frame.height(),
551 video_frame.timestamp_us() * 1000, &cropped_width,
552 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100553 VideoFrame adapted_frame =
554 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200555 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100556 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200557 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100558 .set_timestamp_ms(99)
559 .set_rotation(kVideoRotation_0)
560 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100561 if (video_frame.has_update_rect()) {
562 adapted_frame.set_update_rect(
563 video_frame.update_rect().ScaleWithFrame(
564 video_frame.width(), video_frame.height(), 0, 0,
565 video_frame.width(), video_frame.height(), out_width,
566 out_height));
567 }
sprangc5d62e22017-04-02 23:53:04 -0700568 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800569 last_width_.emplace(adapted_frame.width());
570 last_height_.emplace(adapted_frame.height());
571 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200572 last_width_ = absl::nullopt;
573 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700574 }
sprangb1ca0732017-02-01 08:38:12 -0800575 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000576 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800577 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800578 last_width_.emplace(video_frame.width());
579 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800580 }
581 }
582
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200583 void OnOutputFormatRequest(int width, int height) {
584 absl::optional<std::pair<int, int>> target_aspect_ratio =
585 std::make_pair(width, height);
586 absl::optional<int> max_pixel_count = width * height;
587 absl::optional<int> max_fps;
588 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
589 max_fps);
590 }
591
sprangb1ca0732017-02-01 08:38:12 -0800592 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
593 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200594 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100595 rtc::VideoSinkWants prev_wants = sink_wants_locked();
596 bool did_adapt =
597 prev_wants.max_pixel_count != wants.max_pixel_count ||
598 prev_wants.target_pixel_count != wants.target_pixel_count ||
599 prev_wants.max_framerate_fps != wants.max_framerate_fps;
600 if (did_adapt) {
601 last_wants_ = prev_wants;
602 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100603 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200604 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800605 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200606
Erik Språng5e13d052022-08-02 11:42:49 +0200607 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
608
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200609 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800610 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200611 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
612 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200613 absl::optional<int> last_width_;
614 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200615 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800616};
sprangc5d62e22017-04-02 23:53:04 -0700617
Niels Möller213618e2018-07-24 09:29:58 +0200618// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700619class MockableSendStatisticsProxy : public SendStatisticsProxy {
620 public:
621 MockableSendStatisticsProxy(Clock* clock,
622 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100623 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200624 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100625 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700626
627 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200628 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700629 if (mock_stats_)
630 return *mock_stats_;
631 return SendStatisticsProxy::GetStats();
632 }
633
Niels Möller213618e2018-07-24 09:29:58 +0200634 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200635 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200636 if (mock_stats_)
637 return mock_stats_->input_frame_rate;
638 return SendStatisticsProxy::GetInputFrameRate();
639 }
sprangc5d62e22017-04-02 23:53:04 -0700640 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200641 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700642 mock_stats_.emplace(stats);
643 }
644
645 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200646 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700647 mock_stats_.reset();
648 }
649
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
651 on_frame_dropped_ = std::move(callback);
652 }
653
sprangc5d62e22017-04-02 23:53:04 -0700654 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200655 void OnFrameDropped(DropReason reason) override {
656 SendStatisticsProxy::OnFrameDropped(reason);
657 if (on_frame_dropped_)
658 on_frame_dropped_(reason);
659 }
660
Markus Handella3765182020-07-08 13:13:32 +0200661 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200662 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200663 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700664};
665
Markus Handellb4e96d42021-11-05 12:00:55 +0100666class SimpleVideoStreamEncoderFactory {
667 public:
668 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
669 public:
670 using VideoStreamEncoder::VideoStreamEncoder;
671 ~AdaptedVideoStreamEncoder() { Stop(); }
672 };
673
Markus Handell8d87c462021-12-16 11:37:16 +0100674 class MockFakeEncoder : public test::FakeEncoder {
675 public:
676 using FakeEncoder::FakeEncoder;
677 MOCK_METHOD(CodecSpecificInfo,
678 EncodeHook,
679 (EncodedImage & encoded_image,
680 rtc::scoped_refptr<EncodedImageBuffer> buffer),
681 (override));
682 };
683
Markus Handellee225432021-11-29 12:35:12 +0100684 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100686 encoder_settings_.bitrate_allocator_factory =
687 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 }
689
Markus Handell818e7fb2021-12-30 13:01:33 +0100690 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100691 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100692 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200693 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100694 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
695 time_controller_.GetClock(),
696 /*number_of_cores=*/1,
697 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200698 std::make_unique<CpuOveruseDetectorProxy>(
699 /*stats_proxy=*/nullptr,
700 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100701 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100702 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100703 kVideoBitrateAllocation,
704 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100705 result->SetSink(&sink_, /*rotation_applied=*/false);
706 return result;
707 }
708
Markus Handell818e7fb2021-12-30 13:01:33 +0100709 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
710 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
711 TaskQueueBase** encoder_queue_ptr = nullptr) {
712 auto encoder_queue =
713 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
714 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
715 if (encoder_queue_ptr)
716 *encoder_queue_ptr = encoder_queue.get();
717 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
718 std::move(encoder_queue));
719 }
720
Markus Handell9a478b52021-11-18 16:07:01 +0100721 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100722 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100723
Markus Handell818e7fb2021-12-30 13:01:33 +0100724 GlobalSimulatedTimeController* GetTimeController() {
725 return &time_controller_;
726 }
727
Markus Handellb4e96d42021-11-05 12:00:55 +0100728 private:
729 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
730 public:
731 ~NullEncoderSink() override = default;
732 void OnEncoderConfigurationChanged(
733 std::vector<VideoStream> streams,
734 bool is_svc,
735 VideoEncoderConfig::ContentType content_type,
736 int min_transmit_bitrate_bps) override {}
737 void OnBitrateAllocationUpdated(
738 const VideoBitrateAllocation& allocation) override {}
739 void OnVideoLayersAllocationUpdated(
740 VideoLayersAllocation allocation) override {}
741 Result OnEncodedImage(
742 const EncodedImage& encoded_image,
743 const CodecSpecificInfo* codec_specific_info) override {
744 return Result(EncodedImageCallback::Result::OK);
745 }
746 };
747
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100748 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200749 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100750 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
751 time_controller_.CreateTaskQueueFactory()};
752 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
753 std::make_unique<MockableSendStatisticsProxy>(
754 time_controller_.GetClock(),
755 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100756 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
757 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100758 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
759 CreateBuiltinVideoBitrateAllocatorFactory();
760 VideoStreamEncoderSettings encoder_settings_{
761 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100762 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
763 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100764 NullEncoderSink sink_;
765};
766
767class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
768 public:
769 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100770 MOCK_METHOD(void,
771 SetZeroHertzModeEnabled,
772 (absl::optional<ZeroHertzModeParams>),
773 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100774 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100775 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
776 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100777 MOCK_METHOD(void,
778 UpdateLayerQualityConvergence,
779 (int spatial_index, bool converged),
780 (override));
781 MOCK_METHOD(void,
782 UpdateLayerStatus,
783 (int spatial_index, bool enabled),
784 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100785 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100786};
787
philipel9b058032020-02-10 11:30:00 +0100788class MockEncoderSelector
789 : public VideoEncoderFactory::EncoderSelectorInterface {
790 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200791 MOCK_METHOD(void,
792 OnCurrentEncoder,
793 (const SdpVideoFormat& format),
794 (override));
795 MOCK_METHOD(absl::optional<SdpVideoFormat>,
796 OnAvailableBitrate,
797 (const DataRate& rate),
798 (override));
philipel6daa3042022-04-11 10:48:28 +0200799 MOCK_METHOD(absl::optional<SdpVideoFormat>,
800 OnResolutionChange,
801 (const RenderResolution& resolution),
802 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200803 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100804};
805
Markus Handell2e0f4f02021-12-21 19:14:58 +0100806class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
807 public:
808 MOCK_METHOD(void,
809 AddOrUpdateSink,
810 (rtc::VideoSinkInterface<VideoFrame>*,
811 const rtc::VideoSinkWants&),
812 (override));
813 MOCK_METHOD(void,
814 RemoveSink,
815 (rtc::VideoSinkInterface<VideoFrame>*),
816 (override));
817 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
818};
819
perkj803d97f2016-11-01 11:45:46 -0700820} // namespace
821
mflodmancc3d4422017-08-03 08:27:51 -0700822class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700823 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700825
mflodmancc3d4422017-08-03 08:27:51 -0700826 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700827 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700828 codec_width_(320),
829 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200830 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200831 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200832 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700833 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700835 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100836 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
837 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200838 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700839
840 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700841 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700842 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200843 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800844 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200845 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200846 video_send_config_.rtp.payload_name = "FAKE";
847 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700848
Per512ecb32016-09-23 15:52:06 +0200849 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200850 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200851 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
852 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
853 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100854 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700855
Niels Möllerf1338562018-04-26 09:51:47 +0200856 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800857 }
858
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100859 void ConfigureEncoder(
860 VideoEncoderConfig video_encoder_config,
861 VideoStreamEncoder::BitrateAllocationCallbackType
862 allocation_callback_type =
863 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200864 kVideoBitrateAllocationWhenScreenSharing,
865 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700866 if (video_stream_encoder_)
867 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100868
869 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
870 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
871 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
872 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
873 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100874 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100875 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
876 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
877 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200878 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200879 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700880 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700881 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200882 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700883 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200884 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700885 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800886 }
887
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100888 void ResetEncoder(const std::string& payload_name,
889 size_t num_streams,
890 size_t num_temporal_layers,
891 unsigned char num_spatial_layers,
892 bool screenshare,
893 VideoStreamEncoder::BitrateAllocationCallbackType
894 allocation_callback_type =
895 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200896 kVideoBitrateAllocationWhenScreenSharing,
897 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200898 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800899
900 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200901 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
902 num_streams, &video_encoder_config);
903 for (auto& layer : video_encoder_config.simulcast_layers) {
904 layer.num_temporal_layers = num_temporal_layers;
905 layer.max_framerate = kDefaultFramerate;
906 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100907 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200908 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700909 video_encoder_config.content_type =
910 screenshare ? VideoEncoderConfig::ContentType::kScreen
911 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700912 if (payload_name == "VP9") {
913 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
914 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200915 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700916 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200917 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
918 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700919 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200920 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
921 num_cores);
perkj26091b12016-09-01 01:17:40 -0700922 }
923
sprang57c2fff2017-01-16 06:24:02 -0800924 VideoFrame CreateFrame(int64_t ntp_time_ms,
925 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200926 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200927 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200928 destruction_event, codec_width_, codec_height_))
929 .set_ntp_time_ms(ntp_time_ms)
930 .set_timestamp_ms(99)
931 .set_rotation(kVideoRotation_0)
932 .build();
perkj26091b12016-09-01 01:17:40 -0700933 }
934
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100935 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
936 rtc::Event* destruction_event,
937 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200938 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200939 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200940 destruction_event, codec_width_, codec_height_))
941 .set_ntp_time_ms(ntp_time_ms)
942 .set_timestamp_ms(99)
943 .set_rotation(kVideoRotation_0)
944 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
945 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100946 }
947
sprang57c2fff2017-01-16 06:24:02 -0800948 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200949 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
950 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200951 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200952 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200953 .set_ntp_time_ms(ntp_time_ms)
954 .set_timestamp_ms(ntp_time_ms)
955 .set_rotation(kVideoRotation_0)
956 .build();
perkj803d97f2016-11-01 11:45:46 -0700957 }
958
Evan Shrubsole895556e2020-10-05 09:15:13 +0200959 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200960 return VideoFrame::Builder()
961 .set_video_frame_buffer(NV12Buffer::Create(width, height))
962 .set_ntp_time_ms(ntp_time_ms)
963 .set_timestamp_ms(ntp_time_ms)
964 .set_rotation(kVideoRotation_0)
965 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200966 }
967
Noah Richards51db4212019-06-12 06:59:12 -0700968 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
969 rtc::Event* destruction_event,
970 int width,
971 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200972 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200973 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200974 destruction_event, width, height))
975 .set_ntp_time_ms(ntp_time_ms)
976 .set_timestamp_ms(99)
977 .set_rotation(kVideoRotation_0)
978 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700979 }
980
Evan Shrubsole895556e2020-10-05 09:15:13 +0200981 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
982 rtc::Event* destruction_event,
983 int width,
984 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200985 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200986 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200987 destruction_event, width, height))
988 .set_ntp_time_ms(ntp_time_ms)
989 .set_timestamp_ms(99)
990 .set_rotation(kVideoRotation_0)
991 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200992 }
993
Noah Richards51db4212019-06-12 06:59:12 -0700994 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
995 rtc::Event* destruction_event) const {
996 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
997 codec_height_);
998 }
999
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001000 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001002 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001003
1004 video_source_.IncomingCapturedFrame(
1005 CreateFrame(1, codec_width_, codec_height_));
1006 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001007 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001008 }
1009
sprang4847ae62017-06-27 07:06:52 -07001010 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1011 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001012 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001013 }
1014
1015 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1016 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001017 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001018 return ok;
1019 }
1020
1021 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1022 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001023 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001024 }
1025
1026 void ExpectDroppedFrame() {
1027 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001028 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001029 }
1030
1031 bool WaitForFrame(int64_t timeout_ms) {
1032 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001033 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001034 return ok;
1035 }
1036
perkj26091b12016-09-01 01:17:40 -07001037 class TestEncoder : public test::FakeEncoder {
1038 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001039 explicit TestEncoder(TimeController* time_controller)
1040 : FakeEncoder(time_controller->GetClock()),
1041 time_controller_(time_controller) {
1042 RTC_DCHECK(time_controller_);
1043 }
perkj26091b12016-09-01 01:17:40 -07001044
Erik Språngaed30702018-11-05 12:57:17 +01001045 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001047 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001048 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001049 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001050 info.scaling_settings = VideoEncoder::ScalingSettings(
1051 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001052 }
1053 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001054 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1055 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001056 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001057 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001058 for (int tid = 0; tid < num_layers; ++tid)
1059 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001060 }
1061 }
Erik Språngaed30702018-11-05 12:57:17 +01001062 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001063
1064 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001065 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001066 info.apply_alignment_to_all_simulcast_layers =
1067 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001068 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001069 if (is_qp_trusted_.has_value()) {
1070 info.is_qp_trusted = is_qp_trusted_;
1071 }
Erik Språngaed30702018-11-05 12:57:17 +01001072 return info;
kthelgason876222f2016-11-29 01:44:11 -08001073 }
1074
Erik Språngb7cb7b52019-02-26 15:52:33 +01001075 int32_t RegisterEncodeCompleteCallback(
1076 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001077 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 encoded_image_callback_ = callback;
1079 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1080 }
1081
perkjfa10b552016-10-02 23:45:26 -07001082 void ContinueEncode() { continue_encode_event_.Set(); }
1083
1084 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1085 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001086 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001087 EXPECT_EQ(timestamp_, timestamp);
1088 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1089 }
1090
kthelgason2fc52542017-03-03 00:24:41 -08001091 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001093 quality_scaling_ = b;
1094 }
kthelgasonad9010c2017-02-14 00:46:51 -08001095
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001096 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001097 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001098 requested_resolution_alignment_ = requested_resolution_alignment;
1099 }
1100
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001101 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1102 MutexLock lock(&local_mutex_);
1103 apply_alignment_to_all_simulcast_layers_ = b;
1104 }
1105
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001106 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001107 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001108 is_hardware_accelerated_ = is_hardware_accelerated;
1109 }
1110
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001111 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1112 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001114 temporal_layers_supported_[spatial_idx] = supported;
1115 }
1116
Sergey Silkin6456e352019-07-08 17:56:40 +02001117 void SetResolutionBitrateLimits(
1118 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001119 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001120 resolution_bitrate_limits_ = thresholds;
1121 }
1122
sprangfe627f32017-03-29 08:24:59 -07001123 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001124 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001125 force_init_encode_failed_ = force_failure;
1126 }
1127
Niels Möller6bb5ab92019-01-11 11:11:10 +01001128 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001129 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001130 rate_factor_ = rate_factor;
1131 }
1132
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001135 return last_framerate_;
1136 }
1137
Erik Språngd7329ca2019-02-21 21:19:53 +01001138 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001139 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001140 return last_update_rect_;
1141 }
1142
Niels Möller87e2d782019-03-07 10:18:23 +01001143 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001144 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001145 return last_frame_types_;
1146 }
1147
1148 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001149 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001150 keyframe ? VideoFrameType::kVideoFrameKey
1151 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001152 {
Markus Handella3765182020-07-08 13:13:32 +02001153 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001154 last_frame_types_ = frame_type;
1155 }
Niels Möllerb859b322019-03-07 12:40:01 +01001156 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001157 }
1158
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001159 void InjectEncodedImage(const EncodedImage& image,
1160 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001162 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001163 }
1164
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001165 void SetEncodedImageData(
1166 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001167 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001168 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001169 }
1170
Erik Språngd7329ca2019-02-21 21:19:53 +01001171 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001172 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001173 expect_null_frame_ = true;
1174 }
1175
Erik Språng5056af02019-09-02 15:53:11 +02001176 absl::optional<VideoEncoder::RateControlParameters>
1177 GetAndResetLastRateControlSettings() {
1178 auto settings = last_rate_control_settings_;
1179 last_rate_control_settings_.reset();
1180 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001181 }
1182
Henrik Boström56db9ff2021-03-24 09:06:45 +01001183 int GetLastInputWidth() const {
1184 MutexLock lock(&local_mutex_);
1185 return last_input_width_;
1186 }
1187
1188 int GetLastInputHeight() const {
1189 MutexLock lock(&local_mutex_);
1190 return last_input_height_;
1191 }
1192
Evan Shrubsole895556e2020-10-05 09:15:13 +02001193 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1194 MutexLock lock(&local_mutex_);
1195 return last_input_pixel_format_;
1196 }
1197
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001198 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001200 return num_set_rates_;
1201 }
1202
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001203 void SetPreferredPixelFormats(
1204 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1205 pixel_formats) {
1206 MutexLock lock(&local_mutex_);
1207 preferred_pixel_formats_ = std::move(pixel_formats);
1208 }
1209
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001210 void SetIsQpTrusted(absl::optional<bool> trusted) {
1211 MutexLock lock(&local_mutex_);
1212 is_qp_trusted_ = trusted;
1213 }
1214
Erik Språnge4589cb2022-04-06 16:44:30 +02001215 VideoCodecComplexity LastEncoderComplexity() {
1216 MutexLock lock(&local_mutex_);
1217 return last_encoder_complexity_;
1218 }
1219
perkjfa10b552016-10-02 23:45:26 -07001220 private:
perkj26091b12016-09-01 01:17:40 -07001221 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001222 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001223 {
Markus Handella3765182020-07-08 13:13:32 +02001224 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001225 if (expect_null_frame_) {
1226 EXPECT_EQ(input_image.timestamp(), 0u);
1227 EXPECT_EQ(input_image.width(), 1);
1228 last_frame_types_ = *frame_types;
1229 expect_null_frame_ = false;
1230 } else {
1231 EXPECT_GT(input_image.timestamp(), timestamp_);
1232 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1233 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1234 }
perkj26091b12016-09-01 01:17:40 -07001235
1236 timestamp_ = input_image.timestamp();
1237 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001238 last_input_width_ = input_image.width();
1239 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001240 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001241 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001242 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001243 }
Niels Möllerb859b322019-03-07 12:40:01 +01001244 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001245 return result;
1246 }
1247
Niels Möller08ae7ce2020-09-23 15:58:12 +02001248 CodecSpecificInfo EncodeHook(
1249 EncodedImage& encoded_image,
1250 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001251 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001252 {
1253 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001254 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001255 }
1256 MutexLock lock(&local_mutex_);
1257 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001258 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001259 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001260 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001261 }
1262
sprangfe627f32017-03-29 08:24:59 -07001263 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001264 const Settings& settings) override {
1265 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001266
Markus Handella3765182020-07-08 13:13:32 +02001267 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001268 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001269
Erik Språng82fad3d2018-03-21 09:57:23 +01001270 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001271 // Simulate setting up temporal layers, in order to validate the life
1272 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001273 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001274 frame_buffer_controller_ =
1275 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001276 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001277
1278 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1279
Erik Språngb7cb7b52019-02-26 15:52:33 +01001280 if (force_init_encode_failed_) {
1281 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001282 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001284
Erik Språngb7cb7b52019-02-26 15:52:33 +01001285 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001286 return res;
1287 }
1288
Erik Språngb7cb7b52019-02-26 15:52:33 +01001289 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001290 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001291 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1292 initialized_ = EncoderState::kUninitialized;
1293 return FakeEncoder::Release();
1294 }
1295
Erik Språng16cb8f52019-04-12 13:59:09 +02001296 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001297 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001298 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001299 VideoBitrateAllocation adjusted_rate_allocation;
1300 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1301 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001302 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001303 adjusted_rate_allocation.SetBitrate(
1304 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001305 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001306 rate_factor_));
1307 }
1308 }
1309 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001310 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001311 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001312 RateControlParameters adjusted_paramters = parameters;
1313 adjusted_paramters.bitrate = adjusted_rate_allocation;
1314 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001315 }
1316
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001317 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001318 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001319 enum class EncoderState {
1320 kUninitialized,
1321 kInitializationFailed,
1322 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001323 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001324 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001325 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1326 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1327 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1328 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1329 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1330 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001331 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1332 false;
Markus Handella3765182020-07-08 13:13:32 +02001333 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001334 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1335 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001336 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001337 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001338 absl::optional<bool>
1339 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001340 local_mutex_);
1341 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1342 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1343 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001344 absl::optional<VideoEncoder::RateControlParameters>
1345 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001346 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1347 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001348 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001349 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001350 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1351 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001352 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001353 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001354 RTC_GUARDED_BY(local_mutex_);
1355 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001356 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1357 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001358 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1359 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001360 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001361 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1362 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001363 };
1364
mflodmancc3d4422017-08-03 08:27:51 -07001365 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001366 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001367 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1368 : time_controller_(time_controller), test_encoder_(test_encoder) {
1369 RTC_DCHECK(time_controller_);
1370 }
perkj26091b12016-09-01 01:17:40 -07001371
perkj26091b12016-09-01 01:17:40 -07001372 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001373 EXPECT_TRUE(
1374 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1375 }
1376
1377 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1378 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001379 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001380 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001381 return false;
perkj26091b12016-09-01 01:17:40 -07001382 {
Markus Handella3765182020-07-08 13:13:32 +02001383 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001384 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001385 }
1386 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001387 return true;
perkj26091b12016-09-01 01:17:40 -07001388 }
1389
sprangb1ca0732017-02-01 08:38:12 -08001390 void WaitForEncodedFrame(uint32_t expected_width,
1391 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001392 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001393 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001394 }
1395
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001396 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001397 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001398 uint32_t width = 0;
1399 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001400 {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001402 width = last_width_;
1403 height = last_height_;
1404 }
1405 EXPECT_EQ(expected_height, height);
1406 EXPECT_EQ(expected_width, width);
1407 }
1408
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001409 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1410 VideoRotation rotation;
1411 {
Markus Handella3765182020-07-08 13:13:32 +02001412 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001413 rotation = last_rotation_;
1414 }
1415 EXPECT_EQ(expected_rotation, rotation);
1416 }
1417
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001418 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001419
sprangc5d62e22017-04-02 23:53:04 -07001420 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001421 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001422 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001423 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001424 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001425 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001426 }
1427
perkj26091b12016-09-01 01:17:40 -07001428 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001429 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001430 expect_frames_ = false;
1431 }
1432
asaperssonfab67072017-04-04 05:51:49 -07001433 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001434 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001435 return number_of_reconfigurations_;
1436 }
1437
asaperssonfab67072017-04-04 05:51:49 -07001438 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001439 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001440 return min_transmit_bitrate_bps_;
1441 }
1442
Erik Språngd7329ca2019-02-21 21:19:53 +01001443 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001444 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001445 num_expected_layers_ = num_layers;
1446 }
1447
Erik Språngb7cb7b52019-02-26 15:52:33 +01001448 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001449 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001450 return last_capture_time_ms_;
1451 }
1452
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001453 const EncodedImage& GetLastEncodedImage() {
1454 MutexLock lock(&mutex_);
1455 return last_encoded_image_;
1456 }
1457
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001458 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001459 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001460 return std::move(last_encoded_image_data_);
1461 }
1462
Per Kjellanderdcef6412020-10-07 15:09:05 +02001463 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1464 MutexLock lock(&mutex_);
1465 return last_bitrate_allocation_;
1466 }
1467
1468 int number_of_bitrate_allocations() const {
1469 MutexLock lock(&mutex_);
1470 return number_of_bitrate_allocations_;
1471 }
1472
Per Kjellandera9434842020-10-15 17:53:22 +02001473 VideoLayersAllocation GetLastVideoLayersAllocation() {
1474 MutexLock lock(&mutex_);
1475 return last_layers_allocation_;
1476 }
1477
1478 int number_of_layers_allocations() const {
1479 MutexLock lock(&mutex_);
1480 return number_of_layers_allocations_;
1481 }
1482
perkj26091b12016-09-01 01:17:40 -07001483 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001484 Result OnEncodedImage(
1485 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001486 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001487 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001488 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001489 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001490 last_encoded_image_data_ = std::vector<uint8_t>(
1491 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001492 uint32_t timestamp = encoded_image.Timestamp();
1493 if (last_timestamp_ != timestamp) {
1494 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001495 last_width_ = encoded_image._encodedWidth;
1496 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001497 } else {
1498 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001499 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1500 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001501 }
1502 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001503 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001504 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001505 if (num_received_layers_ == num_expected_layers_) {
1506 encoded_frame_event_.Set();
1507 }
sprangb1ca0732017-02-01 08:38:12 -08001508 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001509 }
1510
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001511 void OnEncoderConfigurationChanged(
1512 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001513 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001514 VideoEncoderConfig::ContentType content_type,
1515 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001516 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001517 ++number_of_reconfigurations_;
1518 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1519 }
1520
Per Kjellanderdcef6412020-10-07 15:09:05 +02001521 void OnBitrateAllocationUpdated(
1522 const VideoBitrateAllocation& allocation) override {
1523 MutexLock lock(&mutex_);
1524 ++number_of_bitrate_allocations_;
1525 last_bitrate_allocation_ = allocation;
1526 }
1527
Per Kjellandera9434842020-10-15 17:53:22 +02001528 void OnVideoLayersAllocationUpdated(
1529 VideoLayersAllocation allocation) override {
1530 MutexLock lock(&mutex_);
1531 ++number_of_layers_allocations_;
1532 last_layers_allocation_ = allocation;
1533 rtc::StringBuilder log;
1534 for (const auto& layer : allocation.active_spatial_layers) {
1535 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1536 << "[";
1537 for (const auto target_bitrate :
1538 layer.target_bitrate_per_temporal_layer) {
1539 log << target_bitrate.kbps() << ",";
1540 }
1541 log << "]";
1542 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001543 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001544 }
1545
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001546 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001547 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001548 TestEncoder* test_encoder_;
1549 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001550 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001551 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001552 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001553 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001554 uint32_t last_height_ = 0;
1555 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001556 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001557 size_t num_expected_layers_ = 1;
1558 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001559 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001560 int number_of_reconfigurations_ = 0;
1561 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001562 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1563 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001564 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1565 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001566 };
1567
Sergey Silkin5ee69672019-07-02 14:18:34 +02001568 class VideoBitrateAllocatorProxyFactory
1569 : public VideoBitrateAllocatorFactory {
1570 public:
1571 VideoBitrateAllocatorProxyFactory()
1572 : bitrate_allocator_factory_(
1573 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1574
1575 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1576 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001577 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001578 codec_config_ = codec;
1579 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1580 }
1581
1582 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001583 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001584 return codec_config_;
1585 }
1586
1587 private:
1588 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1589
Markus Handella3765182020-07-08 13:13:32 +02001590 mutable Mutex mutex_;
1591 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001592 };
1593
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001594 Clock* clock() { return time_controller_.GetClock(); }
1595 void AdvanceTime(TimeDelta duration) {
1596 time_controller_.AdvanceTime(duration);
1597 }
1598
1599 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1600
1601 protected:
1602 virtual TaskQueueFactory* GetTaskQueueFactory() {
1603 return time_controller_.GetTaskQueueFactory();
1604 }
1605
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001606 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001607 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001608 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001609 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001610 int codec_width_;
1611 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001612 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001613 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001614 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001615 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001616 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001617 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001618 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001619 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001620};
1621
mflodmancc3d4422017-08-03 08:27:51 -07001622TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001624 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001625 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001626 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001627 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001628 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001629 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001630}
1631
mflodmancc3d4422017-08-03 08:27:51 -07001632TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001633 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001634 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001635 // The encoder will cache up to one frame for a short duration. Adding two
1636 // frames means that the first frame will be dropped and the second frame will
1637 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001638 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001639 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001640 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001641 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001642 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001643
Henrik Boström381d1092020-05-12 18:49:07 +02001644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001646
Sebastian Janssona3177052018-04-10 13:05:49 +02001647 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001648 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001649 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1650
1651 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001652 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001653}
1654
mflodmancc3d4422017-08-03 08:27:51 -07001655TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001657 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001658 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001659 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001660
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001662 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1663
Sebastian Janssona3177052018-04-10 13:05:49 +02001664 // The encoder will cache up to one frame for a short duration. Adding two
1665 // frames means that the first frame will be dropped and the second frame will
1666 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001667 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001668 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001669
Henrik Boström381d1092020-05-12 18:49:07 +02001670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001671 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001672 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001673 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1674 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001675 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001676}
1677
mflodmancc3d4422017-08-03 08:27:51 -07001678TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001680 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001681 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001682 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001683
1684 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001685 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001686
perkja49cbd32016-09-16 07:53:41 -07001687 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001688 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001689 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001690}
1691
mflodmancc3d4422017-08-03 08:27:51 -07001692TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001694 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001695
perkja49cbd32016-09-16 07:53:41 -07001696 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001697 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001698
mflodmancc3d4422017-08-03 08:27:51 -07001699 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001700 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001701 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001702 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1703 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001704}
1705
Markus Handell9a478b52021-11-18 16:07:01 +01001706TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1707 test::FrameForwarder source;
1708 video_stream_encoder_->SetSource(&source,
1709 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001711 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001712
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001713 int dropped_count = 0;
1714 stats_proxy_->SetDroppedFrameCallback(
1715 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1716 ++dropped_count;
1717 });
1718
Markus Handell9a478b52021-11-18 16:07:01 +01001719 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1720 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1721 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001723 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001724}
1725
Henrik Boström56db9ff2021-03-24 09:06:45 +01001726TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001728 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001729
1730 rtc::Event frame_destroyed_event;
1731 video_source_.IncomingCapturedFrame(
1732 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001733 WaitForEncodedFrame(1);
1734 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1735 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001736 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1737 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001738 video_stream_encoder_->Stop();
1739}
1740
Henrik Boström56db9ff2021-03-24 09:06:45 +01001741TEST_F(VideoStreamEncoderTest,
1742 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001743 // Use the cropping factory.
1744 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001745 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001746 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1747 kMaxPayloadLength);
1748 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1749
1750 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001753 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1754 WaitForEncodedFrame(1);
1755 // The encoder will have been configured once.
1756 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001757 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1758 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001759
1760 // Now send in a fake frame that needs to be cropped as the width/height
1761 // aren't divisible by 4 (see CreateEncoderStreams above).
1762 rtc::Event frame_destroyed_event;
1763 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1764 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001765 WaitForEncodedFrame(2);
1766 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1767 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001768 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1769 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001770 video_stream_encoder_->Stop();
1771}
1772
Evan Shrubsole895556e2020-10-05 09:15:13 +02001773TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001776
1777 video_source_.IncomingCapturedFrame(
1778 CreateNV12Frame(1, codec_width_, codec_height_));
1779 WaitForEncodedFrame(1);
1780 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1781 fake_encoder_.GetLastInputPixelFormat());
1782 video_stream_encoder_->Stop();
1783}
1784
Henrik Boström56db9ff2021-03-24 09:06:45 +01001785TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001787 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001788
1789 fake_encoder_.SetPreferredPixelFormats({});
1790
1791 rtc::Event frame_destroyed_event;
1792 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1793 1, &frame_destroyed_event, codec_width_, codec_height_));
1794 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001795 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001796 fake_encoder_.GetLastInputPixelFormat());
1797 video_stream_encoder_->Stop();
1798}
1799
Henrik Boström56db9ff2021-03-24 09:06:45 +01001800TEST_F(VideoStreamEncoderTest,
1801 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001803 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001804
1805 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1806
1807 rtc::Event frame_destroyed_event;
1808 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1809 1, &frame_destroyed_event, codec_width_, codec_height_));
1810 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001811 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001812 fake_encoder_.GetLastInputPixelFormat());
1813 video_stream_encoder_->Stop();
1814}
1815
Henrik Boström56db9ff2021-03-24 09:06:45 +01001816TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001819
1820 // Fake NV12 native frame does not allow mapping to I444.
1821 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1822
1823 rtc::Event frame_destroyed_event;
1824 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1825 1, &frame_destroyed_event, codec_width_, codec_height_));
1826 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001827 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001828 fake_encoder_.GetLastInputPixelFormat());
1829 video_stream_encoder_->Stop();
1830}
1831
Henrik Boström56db9ff2021-03-24 09:06:45 +01001832TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001835
1836 rtc::Event frame_destroyed_event;
1837 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1838 1, &frame_destroyed_event, codec_width_, codec_height_));
1839 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001840 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001841 fake_encoder_.GetLastInputPixelFormat());
1842 video_stream_encoder_->Stop();
1843}
1844
Ying Wang9b881ab2020-02-07 14:29:32 +01001845TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001848 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1849 WaitForEncodedFrame(1);
1850
Henrik Boström381d1092020-05-12 18:49:07 +02001851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001852 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001853 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1854 // frames. Adding two frames means that the first frame will be dropped and
1855 // the second frame will be sent to the encoder.
1856 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1857 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1858 WaitForEncodedFrame(3);
1859 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1860 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1861 WaitForEncodedFrame(5);
1862 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1863 video_stream_encoder_->Stop();
1864}
1865
mflodmancc3d4422017-08-03 08:27:51 -07001866TEST_F(VideoStreamEncoderTest,
1867 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001869 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001870 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001871
1872 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001873 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001874 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001875 // The encoder will have been configured once when the first frame is
1876 // received.
1877 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001878
1879 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001880 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001881 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001883 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001884
1885 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001886 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001887 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001888 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001889 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001890
mflodmancc3d4422017-08-03 08:27:51 -07001891 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001892}
1893
mflodmancc3d4422017-08-03 08:27:51 -07001894TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001897
1898 // Capture a frame and wait for it to synchronize with the encoder thread.
1899 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001900 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001901 // The encoder will have been configured once.
1902 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001903 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1904 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001905
1906 codec_width_ *= 2;
1907 codec_height_ *= 2;
1908 // Capture a frame with a higher resolution and wait for it to synchronize
1909 // with the encoder thread.
1910 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001911 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001912 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1913 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001914 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001915
mflodmancc3d4422017-08-03 08:27:51 -07001916 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001917}
1918
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001919TEST_F(VideoStreamEncoderTest,
1920 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001921 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001922 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001923
1924 // Capture a frame and wait for it to synchronize with the encoder thread.
1925 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1926 WaitForEncodedFrame(1);
1927
1928 VideoEncoderConfig video_encoder_config;
1929 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1930 // Changing the max payload data length recreates encoder.
1931 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1932 kMaxPayloadLength / 2);
1933
1934 // Capture a frame and wait for it to synchronize with the encoder thread.
1935 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1936 WaitForEncodedFrame(2);
1937 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1938
1939 video_stream_encoder_->Stop();
1940}
1941
Sergey Silkin5ee69672019-07-02 14:18:34 +02001942TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001945
1946 VideoEncoderConfig video_encoder_config;
1947 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001948 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1949 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001950 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1951 kMaxPayloadLength);
1952
1953 // Capture a frame and wait for it to synchronize with the encoder thread.
1954 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1955 WaitForEncodedFrame(1);
1956 // The encoder will have been configured once when the first frame is
1957 // received.
1958 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001959 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001960 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001961 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001962 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1963
Sergey Silkin6456e352019-07-08 17:56:40 +02001964 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1965 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001966 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1967 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001968 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1969 kMaxPayloadLength);
1970
1971 // Capture a frame and wait for it to synchronize with the encoder thread.
1972 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1973 WaitForEncodedFrame(2);
1974 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1975 // Bitrate limits have changed - rate allocator should be reconfigured,
1976 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001977 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001978 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001979 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001980 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001981 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001982
1983 video_stream_encoder_->Stop();
1984}
1985
Sergey Silkin6456e352019-07-08 17:56:40 +02001986TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001987 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001989 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001990
Sergey Silkincd02eba2020-01-20 14:48:40 +01001991 const uint32_t kMinEncBitrateKbps = 100;
1992 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001993 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001994 /*frame_size_pixels=*/codec_width_ * codec_height_,
1995 /*min_start_bitrate_bps=*/0,
1996 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1997 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001998 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1999
Sergey Silkincd02eba2020-01-20 14:48:40 +01002000 VideoEncoderConfig video_encoder_config;
2001 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2002 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2003 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2004 (kMinEncBitrateKbps + 1) * 1000;
2005 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2006 kMaxPayloadLength);
2007
2008 // When both encoder and app provide bitrate limits, the intersection of
2009 // provided sets should be used.
2010 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2011 WaitForEncodedFrame(1);
2012 EXPECT_EQ(kMaxEncBitrateKbps,
2013 bitrate_allocator_factory_.codec_config().maxBitrate);
2014 EXPECT_EQ(kMinEncBitrateKbps + 1,
2015 bitrate_allocator_factory_.codec_config().minBitrate);
2016
2017 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2018 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2019 (kMinEncBitrateKbps - 1) * 1000;
2020 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2021 kMaxPayloadLength);
2022 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002024 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002026 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002027 bitrate_allocator_factory_.codec_config().minBitrate);
2028
Sergey Silkincd02eba2020-01-20 14:48:40 +01002029 video_stream_encoder_->Stop();
2030}
2031
2032TEST_F(VideoStreamEncoderTest,
2033 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002035 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002036
2037 const uint32_t kMinAppBitrateKbps = 100;
2038 const uint32_t kMaxAppBitrateKbps = 200;
2039 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2040 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2041 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2042 /*frame_size_pixels=*/codec_width_ * codec_height_,
2043 /*min_start_bitrate_bps=*/0,
2044 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2045 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2046 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2047
2048 VideoEncoderConfig video_encoder_config;
2049 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2050 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2051 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2052 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002053 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2054 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002055
Sergey Silkincd02eba2020-01-20 14:48:40 +01002056 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2057 WaitForEncodedFrame(1);
2058 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002060 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002062
2063 video_stream_encoder_->Stop();
2064}
2065
2066TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002067 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002069 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002070
2071 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002073 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002075 fake_encoder_.SetResolutionBitrateLimits(
2076 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2077
2078 VideoEncoderConfig video_encoder_config;
2079 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2080 video_encoder_config.max_bitrate_bps = 0;
2081 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2082 kMaxPayloadLength);
2083
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002084 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002085 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2086 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2088 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2090 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2091
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002092 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002093 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2094 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002095 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2096 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002097 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2098 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2099
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002100 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002101 // encoder for 360p should be used.
2102 video_source_.IncomingCapturedFrame(
2103 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2104 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002105 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2106 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002107 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2108 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2109
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002110 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002111 // ignored.
2112 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2113 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002114 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2115 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002116 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2117 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002118 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2119 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002120 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2121 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2122
2123 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2124 // for 270p should be used.
2125 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2126 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002127 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2128 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002129 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2130 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2131
2132 video_stream_encoder_->Stop();
2133}
2134
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002135TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002137 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002138
2139 VideoEncoderConfig video_encoder_config;
2140 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2141 video_encoder_config.max_bitrate_bps = 0;
2142 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2143 kMaxPayloadLength);
2144
2145 // Encode 720p frame to get the default encoder target bitrate.
2146 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2147 WaitForEncodedFrame(1);
2148 const uint32_t kDefaultTargetBitrateFor720pKbps =
2149 bitrate_allocator_factory_.codec_config()
2150 .simulcastStream[0]
2151 .targetBitrate;
2152
2153 // Set the max recommended encoder bitrate to something lower than the default
2154 // target bitrate.
2155 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2156 1280 * 720, 10 * 1000, 10 * 1000,
2157 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2158 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2159
2160 // Change resolution to trigger encoder reinitialization.
2161 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2162 WaitForEncodedFrame(2);
2163 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2164 WaitForEncodedFrame(3);
2165
2166 // Ensure the target bitrate is capped by the max bitrate.
2167 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2168 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2169 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2170 .simulcastStream[0]
2171 .targetBitrate *
2172 1000,
2173 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2174
2175 video_stream_encoder_->Stop();
2176}
2177
Åsa Perssona7e34d32021-01-20 15:36:13 +01002178TEST_F(VideoStreamEncoderTest,
2179 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2180 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2181 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2182 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2183 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2184 fake_encoder_.SetResolutionBitrateLimits(
2185 {kEncoderLimits270p, kEncoderLimits360p});
2186
2187 // Two streams, highest stream active.
2188 VideoEncoderConfig config;
2189 const int kNumStreams = 2;
2190 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2191 config.max_bitrate_bps = 0;
2192 config.simulcast_layers[0].active = false;
2193 config.simulcast_layers[1].active = true;
2194 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002195 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196 "VP8", /*max qp*/ 56, /*screencast*/ false,
2197 /*screenshare enabled*/ false);
2198 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2199
2200 // The encoder bitrate limits for 270p should be used.
2201 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002202 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002208
2209 // The encoder bitrate limits for 360p should be used.
2210 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002211 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002212 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002215 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216
2217 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2218 video_source_.IncomingCapturedFrame(
2219 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002221 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225
2226 // Resolution higher than 360p. Encoder limits should be ignored.
2227 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002228 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002229 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237
2238 // Resolution lower than 270p. The encoder limits for 270p should be used.
2239 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002240 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002244 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002245
2246 video_stream_encoder_->Stop();
2247}
2248
2249TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002250 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2251 // Two streams, highest stream active.
2252 VideoEncoderConfig config;
2253 const int kNumStreams = 2;
2254 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2255 config.max_bitrate_bps = 0;
2256 config.simulcast_layers[0].active = false;
2257 config.simulcast_layers[1].active = true;
2258 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002259 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002260 "VP8", /*max qp*/ 56, /*screencast*/ false,
2261 /*screenshare enabled*/ false);
2262 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2263
2264 // Default bitrate limits for 270p should be used.
2265 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2266 kDefaultLimits270p =
2267 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002268 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002269 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002270 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002271 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002272 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002273 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002274 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002276
2277 // Default bitrate limits for 360p should be used.
2278 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2279 kDefaultLimits360p =
2280 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002281 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002282 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002283 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002284 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002285 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002286 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002287 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002288
2289 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2290 video_source_.IncomingCapturedFrame(
2291 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002292 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002293 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002295 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002296 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002297
2298 // Default bitrate limits for 540p should be used.
2299 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2300 kDefaultLimits540p =
2301 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002302 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002303 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002304 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002305 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002306 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002307 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002308 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002309
2310 video_stream_encoder_->Stop();
2311}
2312
2313TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002314 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2315 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2316 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2317 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2318 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2319 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2320 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2321 fake_encoder_.SetResolutionBitrateLimits(
2322 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2323
2324 // Three streams, middle stream active.
2325 VideoEncoderConfig config;
2326 const int kNumStreams = 3;
2327 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2328 config.simulcast_layers[0].active = false;
2329 config.simulcast_layers[1].active = true;
2330 config.simulcast_layers[2].active = false;
2331 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002332 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002333 "VP8", /*max qp*/ 56, /*screencast*/ false,
2334 /*screenshare enabled*/ false);
2335 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2336
2337 // The encoder bitrate limits for 360p should be used.
2338 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002339 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002340 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002342 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002345
2346 // The encoder bitrate limits for 270p should be used.
2347 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002348 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002350 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002351 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002352 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002353
2354 video_stream_encoder_->Stop();
2355}
2356
2357TEST_F(VideoStreamEncoderTest,
2358 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2359 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2360 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2361 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2362 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2363 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2364 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2365 fake_encoder_.SetResolutionBitrateLimits(
2366 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2367
2368 // Three streams, lowest stream active.
2369 VideoEncoderConfig config;
2370 const int kNumStreams = 3;
2371 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2372 config.simulcast_layers[0].active = true;
2373 config.simulcast_layers[1].active = false;
2374 config.simulcast_layers[2].active = false;
2375 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002376 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002377 "VP8", /*max qp*/ 56, /*screencast*/ false,
2378 /*screenshare enabled*/ false);
2379 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2380
2381 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2382 // on lowest stream, limits for 270p should not be used
2383 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002384 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002385 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002386 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002387 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002388 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002389 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002390
2391 video_stream_encoder_->Stop();
2392}
2393
2394TEST_F(VideoStreamEncoderTest,
2395 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2396 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2397 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2398 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2399 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2400 fake_encoder_.SetResolutionBitrateLimits(
2401 {kEncoderLimits270p, kEncoderLimits360p});
2402 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2403
2404 // Two streams, highest stream active.
2405 VideoEncoderConfig config;
2406 const int kNumStreams = 2;
2407 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2408 config.simulcast_layers[0].active = false;
2409 config.simulcast_layers[1].active = true;
2410 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2411 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002412 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002413 "VP8", /*max qp*/ 56, /*screencast*/ false,
2414 /*screenshare enabled*/ false);
2415 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2416
2417 // The encoder bitrate limits for 270p should be used.
2418 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002419 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002420 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002421 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002424 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425
2426 // The max configured bitrate is less than the encoder limit for 360p.
2427 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002428 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002429 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002430 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002431 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002432 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002433
2434 video_stream_encoder_->Stop();
2435}
2436
mflodmancc3d4422017-08-03 08:27:51 -07002437TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002438 EXPECT_TRUE(video_source_.has_sinks());
2439 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002441 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002442 EXPECT_FALSE(video_source_.has_sinks());
2443 EXPECT_TRUE(new_video_source.has_sinks());
2444
mflodmancc3d4422017-08-03 08:27:51 -07002445 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002446}
2447
mflodmancc3d4422017-08-03 08:27:51 -07002448TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002449 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002451 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002452 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002453}
2454
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002455class ResolutionAlignmentTest
2456 : public VideoStreamEncoderTest,
2457 public ::testing::WithParamInterface<
2458 ::testing::tuple<int, std::vector<double>>> {
2459 public:
2460 ResolutionAlignmentTest()
2461 : requested_alignment_(::testing::get<0>(GetParam())),
2462 scale_factors_(::testing::get<1>(GetParam())) {}
2463
2464 protected:
2465 const int requested_alignment_;
2466 const std::vector<double> scale_factors_;
2467};
2468
2469INSTANTIATE_TEST_SUITE_P(
2470 AlignmentAndScaleFactors,
2471 ResolutionAlignmentTest,
2472 ::testing::Combine(
2473 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2474 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2475 std::vector<double>{-1.0, -1.0},
2476 std::vector<double>{-1.0, -1.0, -1.0},
2477 std::vector<double>{4.0, 2.0, 1.0},
2478 std::vector<double>{9999.0, -1.0, 1.0},
2479 std::vector<double>{3.99, 2.01, 1.0},
2480 std::vector<double>{4.9, 1.7, 1.25},
2481 std::vector<double>{10.0, 4.0, 3.0},
2482 std::vector<double>{1.75, 3.5},
2483 std::vector<double>{1.5, 2.5},
2484 std::vector<double>{1.3, 1.0})));
2485
2486TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2487 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002488 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002489 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2490 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2491
2492 // Fill config with the scaling factor by which to reduce encoding size.
2493 const int num_streams = scale_factors_.size();
2494 VideoEncoderConfig config;
2495 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2496 for (int i = 0; i < num_streams; ++i) {
2497 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2498 }
2499 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002500 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002501 "VP8", /*max qp*/ 56, /*screencast*/ false,
2502 /*screenshare enabled*/ false);
2503 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2504
Henrik Boström381d1092020-05-12 18:49:07 +02002505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002506 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2507 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002508 // Wait for all layers before triggering event.
2509 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002510
2511 // On the 1st frame, we should have initialized the encoder and
2512 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002513 int64_t timestamp_ms = kFrameIntervalMs;
2514 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2515 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002516 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002517
2518 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2519 // (It's up the to the encoder to potentially drop the previous frame,
2520 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002521 timestamp_ms += kFrameIntervalMs;
2522 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2523 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002524 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002525
Asa Persson606d3cb2021-10-04 10:07:11 +02002526 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002527 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2528 // Frame size should be a multiple of the requested alignment.
2529 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2530 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2531 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2532 // Aspect ratio should match.
2533 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2534 codec.height * codec.simulcastStream[i].width);
2535 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002536
2537 video_stream_encoder_->Stop();
2538}
2539
Jonathan Yubc771b72017-12-08 17:04:29 -08002540TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2541 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002542 const int kWidth = 1280;
2543 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002544
2545 // We rely on the automatic resolution adaptation, but we handle framerate
2546 // adaptation manually by mocking the stats proxy.
2547 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002548
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002549 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002551 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002552 video_stream_encoder_->SetSource(&video_source_,
2553 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002554 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002555 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2558
Jonathan Yubc771b72017-12-08 17:04:29 -08002559 // Adapt down as far as possible.
2560 rtc::VideoSinkWants last_wants;
2561 int64_t t = 1;
2562 int loop_count = 0;
2563 do {
2564 ++loop_count;
2565 last_wants = video_source_.sink_wants();
2566
2567 // Simulate the framerate we've been asked to adapt to.
2568 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2569 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2570 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2571 mock_stats.input_frame_rate = fps;
2572 stats_proxy_->SetMockStats(mock_stats);
2573
2574 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2575 sink_.WaitForEncodedFrame(t);
2576 t += frame_interval_ms;
2577
mflodmancc3d4422017-08-03 08:27:51 -07002578 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002579 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002580 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002581 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2582 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002583 } while (video_source_.sink_wants().max_pixel_count <
2584 last_wants.max_pixel_count ||
2585 video_source_.sink_wants().max_framerate_fps <
2586 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002587
Jonathan Yubc771b72017-12-08 17:04:29 -08002588 // Verify that we've adapted all the way down.
2589 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002590 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2592 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002593 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002594 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2595 *video_source_.last_sent_height());
2596 EXPECT_EQ(kMinBalancedFramerateFps,
2597 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002598
Jonathan Yubc771b72017-12-08 17:04:29 -08002599 // Adapt back up the same number of times we adapted down.
2600 for (int i = 0; i < loop_count - 1; ++i) {
2601 last_wants = video_source_.sink_wants();
2602
2603 // Simulate the framerate we've been asked to adapt to.
2604 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2605 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2606 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2607 mock_stats.input_frame_rate = fps;
2608 stats_proxy_->SetMockStats(mock_stats);
2609
2610 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2611 sink_.WaitForEncodedFrame(t);
2612 t += frame_interval_ms;
2613
Henrik Boström91aa7322020-04-28 12:24:33 +02002614 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002615 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002616 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002617 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2618 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002619 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2620 last_wants.max_pixel_count ||
2621 video_source_.sink_wants().max_framerate_fps >
2622 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002623 }
2624
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002625 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002626 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2629 EXPECT_EQ((loop_count - 1) * 2,
2630 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002631
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002633}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002634
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002635TEST_F(VideoStreamEncoderTest,
2636 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002637 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2638 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002639 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002640
2641 const int kFrameWidth = 1280;
2642 const int kFrameHeight = 720;
2643
2644 int64_t ntp_time = kFrameIntervalMs;
2645
2646 // Force an input frame rate to be available, or the adaptation call won't
2647 // know what framerate to adapt form.
2648 const int kInputFps = 30;
2649 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2650 stats.input_frame_rate = kInputFps;
2651 stats_proxy_->SetMockStats(stats);
2652
2653 video_source_.set_adaptation_enabled(true);
2654 video_stream_encoder_->SetSource(
2655 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002656 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002657 video_source_.IncomingCapturedFrame(
2658 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2659 sink_.WaitForEncodedFrame(ntp_time);
2660 ntp_time += kFrameIntervalMs;
2661
2662 // Trigger CPU overuse.
2663 video_stream_encoder_->TriggerCpuOveruse();
2664 video_source_.IncomingCapturedFrame(
2665 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2666 sink_.WaitForEncodedFrame(ntp_time);
2667 ntp_time += kFrameIntervalMs;
2668
2669 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2670 EXPECT_EQ(std::numeric_limits<int>::max(),
2671 video_source_.sink_wants().max_pixel_count);
2672 // Some framerate constraint should be set.
2673 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2674 EXPECT_LT(restricted_fps, kInputFps);
2675 video_source_.IncomingCapturedFrame(
2676 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2677 sink_.WaitForEncodedFrame(ntp_time);
2678 ntp_time += 100;
2679
Henrik Boström2671dac2020-05-19 16:29:09 +02002680 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002681 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2682 // Give the encoder queue time to process the change in degradation preference
2683 // by waiting for an encoded frame.
2684 video_source_.IncomingCapturedFrame(
2685 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2686 sink_.WaitForEncodedFrame(ntp_time);
2687 ntp_time += kFrameIntervalMs;
2688
2689 video_stream_encoder_->TriggerQualityLow();
2690 video_source_.IncomingCapturedFrame(
2691 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2692 sink_.WaitForEncodedFrame(ntp_time);
2693 ntp_time += kFrameIntervalMs;
2694
2695 // Some resolution constraint should be set.
2696 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2697 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2698 kFrameWidth * kFrameHeight);
2699 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2700
2701 int pixel_count = video_source_.sink_wants().max_pixel_count;
2702 // Triggering a CPU underuse should not change the sink wants since it has
2703 // not been overused for resolution since we changed degradation preference.
2704 video_stream_encoder_->TriggerCpuUnderuse();
2705 video_source_.IncomingCapturedFrame(
2706 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2707 sink_.WaitForEncodedFrame(ntp_time);
2708 ntp_time += kFrameIntervalMs;
2709 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2710 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2711
Evan Shrubsole64469032020-06-11 10:45:29 +02002712 // Change the degradation preference back. CPU underuse should not adapt since
2713 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002714 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002715 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2716 video_source_.IncomingCapturedFrame(
2717 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2718 sink_.WaitForEncodedFrame(ntp_time);
2719 ntp_time += 100;
2720 // Resolution adaptations is gone after changing degradation preference.
2721 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2722 EXPECT_EQ(std::numeric_limits<int>::max(),
2723 video_source_.sink_wants().max_pixel_count);
2724 // The fps adaptation from above is now back.
2725 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2726
2727 // Trigger CPU underuse.
2728 video_stream_encoder_->TriggerCpuUnderuse();
2729 video_source_.IncomingCapturedFrame(
2730 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2731 sink_.WaitForEncodedFrame(ntp_time);
2732 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002733 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2734
2735 // Trigger QP underuse, fps should return to normal.
2736 video_stream_encoder_->TriggerQualityHigh();
2737 video_source_.IncomingCapturedFrame(
2738 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2739 sink_.WaitForEncodedFrame(ntp_time);
2740 ntp_time += kFrameIntervalMs;
2741 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002742
2743 video_stream_encoder_->Stop();
2744}
2745
mflodmancc3d4422017-08-03 08:27:51 -07002746TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002747 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002748 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002749 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002750
sprangc5d62e22017-04-02 23:53:04 -07002751 const int kFrameWidth = 1280;
2752 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002753
Åsa Persson8c1bf952018-09-13 10:42:19 +02002754 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002755
kthelgason5e13d412016-12-01 03:59:51 -08002756 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002757 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002758 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002759 frame_timestamp += kFrameIntervalMs;
2760
perkj803d97f2016-11-01 11:45:46 -07002761 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002763 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002764 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002765 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002766 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002767
asapersson0944a802017-04-07 00:57:58 -07002768 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002769 // wanted resolution.
2770 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2771 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2772 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002773 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002774
2775 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002776 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002777 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002779 // Give the encoder queue time to process the change in degradation preference
2780 // by waiting for an encoded frame.
2781 new_video_source.IncomingCapturedFrame(
2782 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2783 sink_.WaitForEncodedFrame(frame_timestamp);
2784 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002785 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002786 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002787
sprangc5d62e22017-04-02 23:53:04 -07002788 // Force an input frame rate to be available, or the adaptation call won't
2789 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002790 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002791 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002792 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002793 stats_proxy_->SetMockStats(stats);
2794
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002796 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002797 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002798 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002799 frame_timestamp += kFrameIntervalMs;
2800
2801 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002802 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002803 EXPECT_EQ(std::numeric_limits<int>::max(),
2804 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002805 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002806
asapersson02465b82017-04-10 01:12:52 -07002807 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002808 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2809 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002810 // Give the encoder queue time to process the change in degradation preference
2811 // by waiting for an encoded frame.
2812 new_video_source.IncomingCapturedFrame(
2813 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2814 sink_.WaitForEncodedFrame(frame_timestamp);
2815 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002816 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002817
mflodmancc3d4422017-08-03 08:27:51 -07002818 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002819 new_video_source.IncomingCapturedFrame(
2820 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002821 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002822 frame_timestamp += kFrameIntervalMs;
2823
2824 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002825 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002826
2827 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002828 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002829 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002830 // Give the encoder queue time to process the change in degradation preference
2831 // by waiting for an encoded frame.
2832 new_video_source.IncomingCapturedFrame(
2833 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2834 sink_.WaitForEncodedFrame(frame_timestamp);
2835 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002836 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2837 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002838 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002839 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002840
2841 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002842 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002843 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002844 // Give the encoder queue time to process the change in degradation preference
2845 // by waiting for an encoded frame.
2846 new_video_source.IncomingCapturedFrame(
2847 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2848 sink_.WaitForEncodedFrame(frame_timestamp);
2849 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002850 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2851 EXPECT_EQ(std::numeric_limits<int>::max(),
2852 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002853 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002854
mflodmancc3d4422017-08-03 08:27:51 -07002855 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002856}
2857
mflodmancc3d4422017-08-03 08:27:51 -07002858TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002860 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002861
asaperssonfab67072017-04-04 05:51:49 -07002862 const int kWidth = 1280;
2863 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002864 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002865 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002866 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2867 EXPECT_FALSE(stats.bw_limited_resolution);
2868 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2869
2870 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002871 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002872 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002874
2875 stats = stats_proxy_->GetStats();
2876 EXPECT_TRUE(stats.bw_limited_resolution);
2877 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2878
2879 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002880 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002881 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002883
2884 stats = stats_proxy_->GetStats();
2885 EXPECT_FALSE(stats.bw_limited_resolution);
2886 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2887 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2888
mflodmancc3d4422017-08-03 08:27:51 -07002889 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002890}
2891
mflodmancc3d4422017-08-03 08:27:51 -07002892TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002894 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002895
2896 const int kWidth = 1280;
2897 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002898 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002899 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002900 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2901 EXPECT_FALSE(stats.cpu_limited_resolution);
2902 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2903
2904 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002905 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002906 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002907 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002908
2909 stats = stats_proxy_->GetStats();
2910 EXPECT_TRUE(stats.cpu_limited_resolution);
2911 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2912
2913 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002914 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002915 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002917
2918 stats = stats_proxy_->GetStats();
2919 EXPECT_FALSE(stats.cpu_limited_resolution);
2920 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002921 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002922
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002924}
2925
mflodmancc3d4422017-08-03 08:27:51 -07002926TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002929
asaperssonfab67072017-04-04 05:51:49 -07002930 const int kWidth = 1280;
2931 const int kHeight = 720;
2932 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002933 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002934 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002935 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002936 EXPECT_FALSE(stats.cpu_limited_resolution);
2937 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2938
asaperssonfab67072017-04-04 05:51:49 -07002939 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002940 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002941 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002942 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002943 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002944 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002945 EXPECT_TRUE(stats.cpu_limited_resolution);
2946 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2947
2948 // Set new source with adaptation still enabled.
2949 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002951 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002952
asaperssonfab67072017-04-04 05:51:49 -07002953 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002955 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002956 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002957 EXPECT_TRUE(stats.cpu_limited_resolution);
2958 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2959
2960 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 video_stream_encoder_->SetSource(&new_video_source,
2962 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002963
asaperssonfab67072017-04-04 05:51:49 -07002964 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002966 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002967 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002968 EXPECT_FALSE(stats.cpu_limited_resolution);
2969 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2970
2971 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002973 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002974
asaperssonfab67072017-04-04 05:51:49 -07002975 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002976 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002977 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002978 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002979 EXPECT_TRUE(stats.cpu_limited_resolution);
2980 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2981
asaperssonfab67072017-04-04 05:51:49 -07002982 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002983 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002984 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002985 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002986 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002987 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002988 EXPECT_FALSE(stats.cpu_limited_resolution);
2989 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002990 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002991
mflodmancc3d4422017-08-03 08:27:51 -07002992 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002993}
2994
mflodmancc3d4422017-08-03 08:27:51 -07002995TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002998
asaperssonfab67072017-04-04 05:51:49 -07002999 const int kWidth = 1280;
3000 const int kHeight = 720;
3001 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003002 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003003 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003004 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003005 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003006 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003007
3008 // Set new source with adaptation still enabled.
3009 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003010 video_stream_encoder_->SetSource(&new_video_source,
3011 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003012
asaperssonfab67072017-04-04 05:51:49 -07003013 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003014 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003015 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003016 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003017 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003018 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003019
asaperssonfab67072017-04-04 05:51:49 -07003020 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003021 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003022 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003023 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003024 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003025 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003026 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003027 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003028
asaperssonfab67072017-04-04 05:51:49 -07003029 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003030 video_stream_encoder_->SetSource(&new_video_source,
3031 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003032
asaperssonfab67072017-04-04 05:51:49 -07003033 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003034 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003035 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003036 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003037 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003038 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003039
asapersson02465b82017-04-10 01:12:52 -07003040 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003042 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003043
asaperssonfab67072017-04-04 05:51:49 -07003044 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003045 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003046 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003047 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003048 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003049 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3050 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003051
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003053}
3054
mflodmancc3d4422017-08-03 08:27:51 -07003055TEST_F(VideoStreamEncoderTest,
3056 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003058 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003059
3060 const int kWidth = 1280;
3061 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003062 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003063 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003064 video_source_.IncomingCapturedFrame(
3065 CreateFrame(timestamp_ms, kWidth, kHeight));
3066 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3068 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3069 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3070
3071 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003073 timestamp_ms += kFrameIntervalMs;
3074 video_source_.IncomingCapturedFrame(
3075 CreateFrame(timestamp_ms, kWidth, kHeight));
3076 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003077 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3078 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3079 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3080
3081 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003082 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003083 timestamp_ms += kFrameIntervalMs;
3084 video_source_.IncomingCapturedFrame(
3085 CreateFrame(timestamp_ms, kWidth, kHeight));
3086 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003087 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3088 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3089 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3090
Niels Möller4db138e2018-04-19 09:04:13 +02003091 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003092 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003093
3094 VideoEncoderConfig video_encoder_config;
3095 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3096 // Make format different, to force recreation of encoder.
3097 video_encoder_config.video_format.parameters["foo"] = "foo";
3098 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003099 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003100 timestamp_ms += kFrameIntervalMs;
3101 video_source_.IncomingCapturedFrame(
3102 CreateFrame(timestamp_ms, kWidth, kHeight));
3103 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003104 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3105 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3106 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3107
mflodmancc3d4422017-08-03 08:27:51 -07003108 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003109}
3110
mflodmancc3d4422017-08-03 08:27:51 -07003111TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003112 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003113 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003114 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003115
3116 const int kWidth = 1280;
3117 const int kHeight = 720;
3118 int sequence = 1;
3119
3120 // Enable BALANCED preference, no initial limitation.
3121 test::FrameForwarder source;
3122 video_stream_encoder_->SetSource(&source,
3123 webrtc::DegradationPreference::BALANCED);
3124 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3125 WaitForEncodedFrame(sequence++);
3126 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3127 EXPECT_FALSE(stats.cpu_limited_resolution);
3128 EXPECT_FALSE(stats.cpu_limited_framerate);
3129 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3130
3131 // Trigger CPU overuse, should now adapt down.
3132 video_stream_encoder_->TriggerCpuOveruse();
3133 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3134 WaitForEncodedFrame(sequence++);
3135 stats = stats_proxy_->GetStats();
3136 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3137
3138 // Set new degradation preference should clear restrictions since we changed
3139 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003140 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003141 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3142 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3143 WaitForEncodedFrame(sequence++);
3144 stats = stats_proxy_->GetStats();
3145 EXPECT_FALSE(stats.cpu_limited_resolution);
3146 EXPECT_FALSE(stats.cpu_limited_framerate);
3147 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3148
3149 // Force an input frame rate to be available, or the adaptation call won't
3150 // know what framerate to adapt from.
3151 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3152 mock_stats.input_frame_rate = 30;
3153 stats_proxy_->SetMockStats(mock_stats);
3154 video_stream_encoder_->TriggerCpuOveruse();
3155 stats_proxy_->ResetMockStats();
3156 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3157 WaitForEncodedFrame(sequence++);
3158
3159 // We have now adapted once.
3160 stats = stats_proxy_->GetStats();
3161 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3162
3163 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003164 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3165 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003166 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3167 WaitForEncodedFrame(sequence++);
3168 stats = stats_proxy_->GetStats();
3169 EXPECT_FALSE(stats.cpu_limited_resolution);
3170 EXPECT_FALSE(stats.cpu_limited_framerate);
3171 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3172
3173 video_stream_encoder_->Stop();
3174}
3175
3176TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003177 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003179 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003180
asapersson0944a802017-04-07 00:57:58 -07003181 const int kWidth = 1280;
3182 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003183 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003184
asaperssonfab67072017-04-04 05:51:49 -07003185 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003186 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003187 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003188 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003189 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003190 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3191
asapersson02465b82017-04-10 01:12:52 -07003192 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003194 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003195 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003196 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003197 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003198 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003199 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3200
3201 // Set new source with adaptation still enabled.
3202 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003204 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003205
3206 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003207 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003208 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003209 stats = stats_proxy_->GetStats();
3210 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003211 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003212 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3213
sprangc5d62e22017-04-02 23:53:04 -07003214 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003216 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003217 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003218 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003219 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003220 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003221 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003222 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003223 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003224 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3225
sprangc5d62e22017-04-02 23:53:04 -07003226 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003227 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003228 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3229 mock_stats.input_frame_rate = 30;
3230 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003232 stats_proxy_->ResetMockStats();
3233
3234 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003235 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003236 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003237
3238 // Framerate now adapted.
3239 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003240 EXPECT_FALSE(stats.cpu_limited_resolution);
3241 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003242 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3243
3244 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003245 video_stream_encoder_->SetSource(&new_video_source,
3246 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003247 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003248 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003249 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003250
3251 stats = stats_proxy_->GetStats();
3252 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003253 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003254 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3255
3256 // Try to trigger overuse. Should not succeed.
3257 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003258 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003259 stats_proxy_->ResetMockStats();
3260
3261 stats = stats_proxy_->GetStats();
3262 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003263 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003264 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3265
3266 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003268 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003269 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003271 stats = stats_proxy_->GetStats();
3272 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003273 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003274 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003275
3276 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003277 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003278 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003279 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003280 stats = stats_proxy_->GetStats();
3281 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003282 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003283 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3284
3285 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003286 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003287 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003288 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003289 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003290 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003291 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003292 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003293 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003294 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003295 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3296
3297 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003298 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003299 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003300 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003301 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003302 stats = stats_proxy_->GetStats();
3303 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003304 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003305 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003306 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003307
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003309}
3310
mflodmancc3d4422017-08-03 08:27:51 -07003311TEST_F(VideoStreamEncoderTest,
3312 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003313 const int kWidth = 1280;
3314 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003316 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003317
asaperssonfab67072017-04-04 05:51:49 -07003318 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003319 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003320
asaperssonfab67072017-04-04 05:51:49 -07003321 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003322 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003323
asaperssonfab67072017-04-04 05:51:49 -07003324 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003325 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003326
asaperssonfab67072017-04-04 05:51:49 -07003327 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003328 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003329
kthelgason876222f2016-11-29 01:44:11 -08003330 // Expect a scale down.
3331 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003332 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003333
asapersson02465b82017-04-10 01:12:52 -07003334 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003335 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003336 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003337 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003338
asaperssonfab67072017-04-04 05:51:49 -07003339 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003340 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003341 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003342 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003343
asaperssonfab67072017-04-04 05:51:49 -07003344 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003345 EXPECT_EQ(std::numeric_limits<int>::max(),
3346 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003347
asaperssonfab67072017-04-04 05:51:49 -07003348 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003350 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003351 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003352
asapersson02465b82017-04-10 01:12:52 -07003353 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003354 EXPECT_EQ(std::numeric_limits<int>::max(),
3355 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003356
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003358}
3359
mflodmancc3d4422017-08-03 08:27:51 -07003360TEST_F(VideoStreamEncoderTest,
3361 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003362 const int kWidth = 1280;
3363 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003365 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003366
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003367 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003368 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003370 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003371
3372 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003373 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003375 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3376 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3377
3378 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003380 EXPECT_THAT(source.sink_wants(),
3381 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003382 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3383 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3384 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3385
3386 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003387 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003388 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3389 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3391
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003393}
3394
mflodmancc3d4422017-08-03 08:27:51 -07003395TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003396 const int kWidth = 1280;
3397 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003399 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003400
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003401 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003402 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003403 video_stream_encoder_->SetSource(&source,
3404 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003405 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3406 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003407 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003408
3409 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003410 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003411 EXPECT_THAT(source.sink_wants(),
3412 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3414 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3415 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3416
3417 // Trigger adapt down for same input resolution, expect no change.
3418 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3419 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003420 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003421 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down for larger input resolution, expect no change.
3426 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3427 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003428 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003429 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3431 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3432
mflodmancc3d4422017-08-03 08:27:51 -07003433 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003434}
3435
mflodmancc3d4422017-08-03 08:27:51 -07003436TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003437 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3438 const int kWidth = 640;
3439 const int kHeight = 360;
3440 const int64_t kFrameIntervalMs = 150;
3441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003442 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003443
3444 // Enable BALANCED preference, no initial limitation.
3445 AdaptingFrameForwarder source(&time_controller_);
3446 source.set_adaptation_enabled(true);
3447 video_stream_encoder_->SetSource(&source,
3448 webrtc::DegradationPreference::BALANCED);
3449
3450 int64_t timestamp_ms = kFrameIntervalMs;
3451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3452 sink_.WaitForEncodedFrame(kWidth, kHeight);
3453 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3457
3458 // Trigger adapt down, expect reduced fps (640x360@15fps).
3459 video_stream_encoder_->TriggerQualityLow();
3460 timestamp_ms += kFrameIntervalMs;
3461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3462 sink_.WaitForEncodedFrame(timestamp_ms);
3463 EXPECT_THAT(source.sink_wants(),
3464 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3467 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 // Source requests 270p, expect reduced resolution (480x270@15fps).
3470 source.OnOutputFormatRequest(480, 270);
3471 timestamp_ms += kFrameIntervalMs;
3472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3473 WaitForEncodedFrame(480, 270);
3474 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475
3476 // Trigger adapt down, expect reduced fps (480x270@10fps).
3477 video_stream_encoder_->TriggerQualityLow();
3478 timestamp_ms += kFrameIntervalMs;
3479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3480 sink_.WaitForEncodedFrame(timestamp_ms);
3481 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3482 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3484 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3487 source.OnOutputFormatRequest(320, 180);
3488 timestamp_ms += kFrameIntervalMs;
3489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3490 WaitForEncodedFrame(320, 180);
3491 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Trigger adapt down, expect reduced fps (320x180@7fps).
3494 video_stream_encoder_->TriggerQualityLow();
3495 timestamp_ms += kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3497 sink_.WaitForEncodedFrame(timestamp_ms);
3498 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3501 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3502
3503 // Source requests VGA, expect increased resolution (640x360@7fps).
3504 source.OnOutputFormatRequest(640, 360);
3505 timestamp_ms += kFrameIntervalMs;
3506 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3507 WaitForEncodedFrame(timestamp_ms);
3508 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3509
3510 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3511 video_stream_encoder_->TriggerQualityHigh();
3512 timestamp_ms += kFrameIntervalMs;
3513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3514 WaitForEncodedFrame(timestamp_ms);
3515 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3516 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3518 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3519
3520 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3521 video_stream_encoder_->TriggerQualityHigh();
3522 timestamp_ms += kFrameIntervalMs;
3523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3524 WaitForEncodedFrame(timestamp_ms);
3525 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3528 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3529
3530 // Trigger adapt up, expect increased fps (640x360@maxfps).
3531 video_stream_encoder_->TriggerQualityHigh();
3532 timestamp_ms += kFrameIntervalMs;
3533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3534 WaitForEncodedFrame(timestamp_ms);
3535 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3538 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3539
3540 video_stream_encoder_->Stop();
3541}
3542
3543TEST_F(VideoStreamEncoderTest,
3544 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3545 const int kWidth = 1280;
3546 const int kHeight = 720;
3547 const int64_t kFrameIntervalMs = 150;
3548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003549 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003550
3551 // Enable BALANCED preference, no initial limitation.
3552 AdaptingFrameForwarder source(&time_controller_);
3553 source.set_adaptation_enabled(true);
3554 video_stream_encoder_->SetSource(&source,
3555 webrtc::DegradationPreference::BALANCED);
3556
3557 int64_t timestamp_ms = kFrameIntervalMs;
3558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3559 sink_.WaitForEncodedFrame(kWidth, kHeight);
3560 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3566 video_stream_encoder_->TriggerQualityLow();
3567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 sink_.WaitForEncodedFrame(timestamp_ms);
3570 EXPECT_THAT(source.sink_wants(),
3571 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3574 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3575
3576 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3577 video_stream_encoder_->TriggerQualityLow();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 sink_.WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585
3586 // Trigger adapt down, expect reduced fps (640x360@15fps).
3587 video_stream_encoder_->TriggerQualityLow();
3588 timestamp_ms += kFrameIntervalMs;
3589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3590 WaitForEncodedFrame(timestamp_ms);
3591 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3594 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3595
3596 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3597 source.OnOutputFormatRequest(320, 180);
3598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(320, 180);
3601 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3603
3604 // Trigger adapt down, expect reduced fps (320x180@7fps).
3605 video_stream_encoder_->TriggerCpuOveruse();
3606 timestamp_ms += kFrameIntervalMs;
3607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3608 WaitForEncodedFrame(timestamp_ms);
3609 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3612 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3613 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3616
3617 // Source requests HD, expect increased resolution (640x360@7fps).
3618 source.OnOutputFormatRequest(1280, 720);
3619 timestamp_ms += kFrameIntervalMs;
3620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3621 WaitForEncodedFrame(timestamp_ms);
3622 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3624
3625 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3626 video_stream_encoder_->TriggerCpuUnderuse();
3627 timestamp_ms += kFrameIntervalMs;
3628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3629 WaitForEncodedFrame(timestamp_ms);
3630 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3633 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3634 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3635 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3636 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3637
3638 // Trigger adapt up, expect increased fps (640x360@maxfps).
3639 video_stream_encoder_->TriggerQualityHigh();
3640 video_stream_encoder_->TriggerCpuUnderuse();
3641 timestamp_ms += kFrameIntervalMs;
3642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3643 WaitForEncodedFrame(timestamp_ms);
3644 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3646 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3647 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3649 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3650 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3651
3652 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3653 video_stream_encoder_->TriggerQualityHigh();
3654 video_stream_encoder_->TriggerCpuUnderuse();
3655 timestamp_ms += kFrameIntervalMs;
3656 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3657 WaitForEncodedFrame(timestamp_ms);
3658 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3659 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3661 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3663 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3664 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3665
3666 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3667 video_stream_encoder_->TriggerQualityHigh();
3668 video_stream_encoder_->TriggerCpuUnderuse();
3669 timestamp_ms += kFrameIntervalMs;
3670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3671 WaitForEncodedFrame(timestamp_ms);
3672 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3677 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3678 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3679
3680 video_stream_encoder_->Stop();
3681}
3682
3683TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003684 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003685 const int kWidth = 1280;
3686 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003688 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003689
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003690 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003691 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003693 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003694
3695 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003696 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003697 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003698 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3699 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3700
3701 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003702 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003703 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003704 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3705 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3706
mflodmancc3d4422017-08-03 08:27:51 -07003707 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003708}
3709
mflodmancc3d4422017-08-03 08:27:51 -07003710TEST_F(VideoStreamEncoderTest,
3711 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003712 const int kWidth = 1280;
3713 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003716
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003717 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003718 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003719 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003720 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003721
3722 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003723 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003724 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003726 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3727
3728 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003729 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003730 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003735}
3736
mflodmancc3d4422017-08-03 08:27:51 -07003737TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003738 const int kWidth = 1280;
3739 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003741 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003742
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003743 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003744 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003745 video_stream_encoder_->SetSource(&source,
3746 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003747
3748 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3749 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003750 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003751 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3753 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3754
3755 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003756 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3760 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003763}
3764
mflodmancc3d4422017-08-03 08:27:51 -07003765TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003766 const int kWidth = 1280;
3767 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003769 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003770
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003771 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003772 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003773 video_stream_encoder_->SetSource(&source,
3774 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003775
3776 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3777 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003779 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3781 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3782
3783 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003784 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003785 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003786 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3787 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3788 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3789
mflodmancc3d4422017-08-03 08:27:51 -07003790 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003791}
3792
mflodmancc3d4422017-08-03 08:27:51 -07003793TEST_F(VideoStreamEncoderTest,
3794 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003795 const int kWidth = 1280;
3796 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003798 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003799
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003800 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003801 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003802 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003803 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003804 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003805
3806 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003807 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003808 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3810 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3811
3812 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003813 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003814 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003815 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003816 EXPECT_THAT(source.sink_wants(),
3817 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003818 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3819 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3820
3821 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003822 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003823 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003824 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3825 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3826 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3827
mflodmancc3d4422017-08-03 08:27:51 -07003828 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003829}
3830
mflodmancc3d4422017-08-03 08:27:51 -07003831TEST_F(VideoStreamEncoderTest,
3832 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003833 const int kWidth = 1280;
3834 const int kHeight = 720;
3835 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003837 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003838
3839 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3840 stats.input_frame_rate = kInputFps;
3841 stats_proxy_->SetMockStats(stats);
3842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003843 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003844 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3845 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003846 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003847
3848 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003849 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003850 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3851 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003852 EXPECT_THAT(video_source_.sink_wants(),
3853 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003854
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003855 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003856 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003857 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003858 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003859 // Give the encoder queue time to process the change in degradation preference
3860 // by waiting for an encoded frame.
3861 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3862 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003863 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003864
3865 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003866 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003867 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3868 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003869 EXPECT_THAT(new_video_source.sink_wants(),
3870 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003871
3872 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003873 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003874 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003875
mflodmancc3d4422017-08-03 08:27:51 -07003876 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003877}
3878
mflodmancc3d4422017-08-03 08:27:51 -07003879TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003880 const int kWidth = 1280;
3881 const int kHeight = 720;
3882 const size_t kNumFrames = 10;
3883
Henrik Boström381d1092020-05-12 18:49:07 +02003884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003886
asaperssond0de2952017-04-21 01:47:31 -07003887 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003888 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003889 video_source_.set_adaptation_enabled(true);
3890
3891 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3893
3894 int downscales = 0;
3895 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 video_source_.IncomingCapturedFrame(
3897 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3898 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003899
asaperssonfab67072017-04-04 05:51:49 -07003900 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003901 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003902 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003903 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003904
3905 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3906 ++downscales;
3907
3908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3909 EXPECT_EQ(downscales,
3910 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3911 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003912 }
mflodmancc3d4422017-08-03 08:27:51 -07003913 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003914}
3915
mflodmancc3d4422017-08-03 08:27:51 -07003916TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003917 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3918 const int kWidth = 1280;
3919 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003921 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003922
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003923 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003924 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003925 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003926 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003927 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003928
Åsa Persson8c1bf952018-09-13 10:42:19 +02003929 int64_t timestamp_ms = kFrameIntervalMs;
3930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003931 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003932 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003933 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3934 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3935
3936 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003937 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003938 timestamp_ms += kFrameIntervalMs;
3939 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3940 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003941 EXPECT_THAT(source.sink_wants(),
3942 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003943 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3944 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3945
3946 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003947 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003948 timestamp_ms += kFrameIntervalMs;
3949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003950 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003951 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003952 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3953 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3954
3955 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003956 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003957 timestamp_ms += kFrameIntervalMs;
3958 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3959 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003960 EXPECT_THAT(source.sink_wants(),
3961 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003962 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3963 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3964
3965 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003966 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003967 timestamp_ms += kFrameIntervalMs;
3968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003969 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003970 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003971 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3972 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3973
mflodmancc3d4422017-08-03 08:27:51 -07003974 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003975}
3976
mflodmancc3d4422017-08-03 08:27:51 -07003977TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003978 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3979 const int kWidth = 1280;
3980 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003983
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003984 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003985 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003986 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003987 video_stream_encoder_->SetSource(&source,
3988 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003989
Åsa Persson8c1bf952018-09-13 10:42:19 +02003990 int64_t timestamp_ms = kFrameIntervalMs;
3991 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003992 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003993 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003994 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3995 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3996
3997 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003998 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003999 timestamp_ms += kFrameIntervalMs;
4000 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4001 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004002 EXPECT_THAT(source.sink_wants(),
4003 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004004 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4005 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4006
4007 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004008 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004009 timestamp_ms += kFrameIntervalMs;
4010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004011 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004012 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004013 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4014 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4015
4016 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004017 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004018 timestamp_ms += kFrameIntervalMs;
4019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4020 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004021 EXPECT_THAT(source.sink_wants(),
4022 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004023 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4024 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4025
4026 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004027 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004028 timestamp_ms += kFrameIntervalMs;
4029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004030 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004031 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4033 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4034
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004036}
4037
Sergey Silkin41c650b2019-10-14 13:12:19 +02004038TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4039 fake_encoder_.SetResolutionBitrateLimits(
4040 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4041
Henrik Boström381d1092020-05-12 18:49:07 +02004042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004043 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4044 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4045 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4046 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004047
4048 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004049 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050 source.set_adaptation_enabled(true);
4051 video_stream_encoder_->SetSource(
4052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4053
4054 // Insert 720p frame.
4055 int64_t timestamp_ms = kFrameIntervalMs;
4056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4057 WaitForEncodedFrame(1280, 720);
4058
4059 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004061 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4062 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4063 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4064 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004065 video_stream_encoder_->TriggerQualityLow();
4066
4067 // Insert 720p frame. It should be downscaled and encoded.
4068 timestamp_ms += kFrameIntervalMs;
4069 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4070 WaitForEncodedFrame(960, 540);
4071
4072 // Trigger adapt up. Higher resolution should not be requested duo to lack
4073 // of bitrate.
4074 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004075 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004076
4077 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004079 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4080 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4081 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4082 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004083
4084 // Trigger adapt up. Higher resolution should be requested.
4085 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004086 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004087
4088 video_stream_encoder_->Stop();
4089}
4090
4091TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4092 fake_encoder_.SetResolutionBitrateLimits(
4093 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4094
4095 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004097 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4098 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4099 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4100 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004101
4102 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004103 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004104 source.set_adaptation_enabled(true);
4105 video_stream_encoder_->SetSource(
4106 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4107
4108 // Insert 720p frame. It should be dropped and lower resolution should be
4109 // requested.
4110 int64_t timestamp_ms = kFrameIntervalMs;
4111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4112 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004113 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004114
4115 // Insert 720p frame. It should be downscaled and encoded.
4116 timestamp_ms += kFrameIntervalMs;
4117 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4118 WaitForEncodedFrame(960, 540);
4119
4120 video_stream_encoder_->Stop();
4121}
4122
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004123class BalancedDegradationTest : public VideoStreamEncoderTest {
4124 protected:
4125 void SetupTest() {
4126 // Reset encoder for field trials to take effect.
4127 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004128 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004129
4130 // Enable BALANCED preference.
4131 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004132 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4133 }
4134
Asa Persson606d3cb2021-10-04 10:07:11 +02004135 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004137 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004138 }
4139
Åsa Persson45b176f2019-09-30 11:19:05 +02004140 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141 timestamp_ms_ += kFrameIntervalMs;
4142 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004143 }
4144
4145 void InsertFrameAndWaitForEncoded() {
4146 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004147 sink_.WaitForEncodedFrame(timestamp_ms_);
4148 }
4149
4150 const int kWidth = 640; // pixels:640x360=230400
4151 const int kHeight = 360;
4152 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4153 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004154 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004155};
4156
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004157TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004158 test::ScopedKeyValueConfig field_trials(
4159 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004160 "WebRTC-Video-BalancedDegradationSettings/"
4161 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4162 SetupTest();
4163
4164 // Force input frame rate.
4165 const int kInputFps = 24;
4166 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4167 stats.input_frame_rate = kInputFps;
4168 stats_proxy_->SetMockStats(stats);
4169
Åsa Persson45b176f2019-09-30 11:19:05 +02004170 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004171 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004172
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004173 // Trigger adapt down, expect scaled down framerate and resolution,
4174 // since Fps diff (input-requested:0) < threshold.
4175 video_stream_encoder_->TriggerQualityLow();
4176 EXPECT_THAT(source_.sink_wants(),
4177 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004178
4179 video_stream_encoder_->Stop();
4180}
4181
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004182TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004183 test::ScopedKeyValueConfig field_trials(
4184 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004185 "WebRTC-Video-BalancedDegradationSettings/"
4186 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4187 SetupTest();
4188
4189 // Force input frame rate.
4190 const int kInputFps = 25;
4191 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4192 stats.input_frame_rate = kInputFps;
4193 stats_proxy_->SetMockStats(stats);
4194
Åsa Persson45b176f2019-09-30 11:19:05 +02004195 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004196 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004197
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004198 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4199 // Fps diff (input-requested:1) == threshold.
4200 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004201 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004202
4203 video_stream_encoder_->Stop();
4204}
4205
4206TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004207 test::ScopedKeyValueConfig field_trials(
4208 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209 "WebRTC-Video-BalancedDegradationSettings/"
4210 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4211 SetupTest();
4212
4213 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4214
Åsa Persson45b176f2019-09-30 11:19:05 +02004215 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004216 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004217
4218 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4219 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004220 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004221
4222 video_stream_encoder_->Stop();
4223}
4224
Åsa Perssonccfb3402019-09-25 15:13:04 +02004225TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004226 test::ScopedKeyValueConfig field_trials(
4227 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004228 "WebRTC-Video-BalancedDegradationSettings/"
4229 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004230 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004231
Asa Persson606d3cb2021-10-04 10:07:11 +02004232 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4233 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4234 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004235
Åsa Persson45b176f2019-09-30 11:19:05 +02004236 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004237 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004238 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4239
4240 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4241 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004242 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004243 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004244 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4245
4246 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4247 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004248 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004249 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004250 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4251
Åsa Persson30ab0152019-08-27 12:22:33 +02004252 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4253 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004255 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004256 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004257 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4258
4259 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004260 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004261 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004263
Åsa Persson30ab0152019-08-27 12:22:33 +02004264 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004265 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004266 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004267 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004268 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004269 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
4271 video_stream_encoder_->Stop();
4272}
4273
Åsa Perssonccfb3402019-09-25 15:13:04 +02004274TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004276 test::ScopedKeyValueConfig field_trials(
4277 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 "WebRTC-Video-BalancedDegradationSettings/"
4279 "pixels:57600|129600|230400,fps:7|24|24/");
4280 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004281 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004282
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004283 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004284
4285 // Insert frame, expect scaled down:
4286 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4287 InsertFrame();
4288 EXPECT_FALSE(WaitForFrame(1000));
4289 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4290 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4291
4292 // Insert frame, expect scaled down:
4293 // resolution (320x180@24fps).
4294 InsertFrame();
4295 EXPECT_FALSE(WaitForFrame(1000));
4296 EXPECT_LT(source_.sink_wants().max_pixel_count,
4297 source_.last_wants().max_pixel_count);
4298 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4299
4300 // Frame should not be dropped (min pixels per frame reached).
4301 InsertFrameAndWaitForEncoded();
4302
4303 video_stream_encoder_->Stop();
4304}
4305
4306TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004308 test::ScopedKeyValueConfig field_trials(
4309 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004310 "WebRTC-Video-BalancedDegradationSettings/"
4311 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004312 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004313
Asa Persson606d3cb2021-10-04 10:07:11 +02004314 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4315 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4316 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004317
Åsa Persson45b176f2019-09-30 11:19:05 +02004318 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004319 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004320 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4321
4322 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4323 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004324 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004325 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004326 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4327
4328 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4329 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004330 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004331 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004332 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4333
4334 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4335 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004336 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004337 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004338 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4339
Åsa Persson30ab0152019-08-27 12:22:33 +02004340 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4341 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004342 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004343 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4347 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004348 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004349 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004352 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004354 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004355 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004356 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4357
4358 video_stream_encoder_->Stop();
4359}
4360
Åsa Perssonccfb3402019-09-25 15:13:04 +02004361TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004362 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004363 test::ScopedKeyValueConfig field_trials(
4364 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004365 "WebRTC-Video-BalancedDegradationSettings/"
4366 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004367 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004368
Asa Persson606d3cb2021-10-04 10:07:11 +02004369 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4370 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4371 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4372 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4373 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004374
Åsa Persson45b176f2019-09-30 11:19:05 +02004375 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004376 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004377 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
4379 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4380 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004381 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004382 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004383 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4384
4385 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4386 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004387 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004388 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004389 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
4391 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4392 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004393 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004394 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004395 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4396
4397 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4398 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004399 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004400 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4401
4402 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004403 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004404 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004405 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004406 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004407 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4408
4409 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004410 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004411 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004412 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004413 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4414
4415 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004416 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004417 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004418 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004419 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004420 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4421
Åsa Persson1b247f12019-08-14 17:26:39 +02004422 video_stream_encoder_->Stop();
4423}
4424
mflodmancc3d4422017-08-03 08:27:51 -07004425TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004426 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4427 const int kWidth = 1280;
4428 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004430 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004431
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004432 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004433 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004434 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004435 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004436 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004437
Åsa Persson8c1bf952018-09-13 10:42:19 +02004438 int64_t timestamp_ms = kFrameIntervalMs;
4439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004440 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004441 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4445 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4446
4447 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004448 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004452 EXPECT_THAT(source.sink_wants(),
4453 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004454 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4456 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4457 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4458
4459 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004460 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004461 timestamp_ms += kFrameIntervalMs;
4462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4463 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004464 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004465 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4467 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4468 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4469
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004471 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004472 timestamp_ms += kFrameIntervalMs;
4473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4474 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004475 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004478 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004479 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4480
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004482 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004486 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004487 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4490 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4491 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4492
Jonathan Yubc771b72017-12-08 17:04:29 -08004493 // Trigger quality adapt down, expect no change (min resolution reached).
4494 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004495 timestamp_ms += kFrameIntervalMs;
4496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4497 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004498 EXPECT_THAT(source.sink_wants(), FpsMax());
4499 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004500 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4501 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4502 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4503 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4504
Evan Shrubsole64469032020-06-11 10:45:29 +02004505 // Trigger quality adapt up, expect upscaled resolution (480x270).
4506 video_stream_encoder_->TriggerQualityHigh();
4507 timestamp_ms += kFrameIntervalMs;
4508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4509 WaitForEncodedFrame(timestamp_ms);
4510 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4511 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4513 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4514 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4515
4516 // Trigger quality and cpu adapt up since both are most limited, expect
4517 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004518 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004519 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004520 timestamp_ms += kFrameIntervalMs;
4521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004523 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4526 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004527 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004528
Evan Shrubsole64469032020-06-11 10:45:29 +02004529 // Trigger quality and cpu adapt up since both are most limited, expect
4530 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004531 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004532 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004533 timestamp_ms += kFrameIntervalMs;
4534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004536 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004537 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004538 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004540 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4541 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004542
Evan Shrubsole64469032020-06-11 10:45:29 +02004543 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4544 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004545 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004546 timestamp_ms += kFrameIntervalMs;
4547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004549 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004550 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004552 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004553 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004554
4555 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004556 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004557 timestamp_ms += kFrameIntervalMs;
4558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004559 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004560 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004561 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004562 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004564 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004565 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004566
mflodmancc3d4422017-08-03 08:27:51 -07004567 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004568}
4569
mflodmancc3d4422017-08-03 08:27:51 -07004570TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004571 const int kWidth = 640;
4572 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004573
Henrik Boström381d1092020-05-12 18:49:07 +02004574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004575 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004576
perkj803d97f2016-11-01 11:45:46 -07004577 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004578 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004579 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004580 }
4581
mflodmancc3d4422017-08-03 08:27:51 -07004582 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004583 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004584 video_source_.IncomingCapturedFrame(CreateFrame(
4585 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004586 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004587 }
4588
mflodmancc3d4422017-08-03 08:27:51 -07004589 video_stream_encoder_->Stop();
4590 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004591 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004592
Ying Wangef3998f2019-12-09 13:06:53 +01004593 EXPECT_METRIC_EQ(
4594 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4595 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004596 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4597}
4598
mflodmancc3d4422017-08-03 08:27:51 -07004599TEST_F(VideoStreamEncoderTest,
4600 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004603 const int kWidth = 640;
4604 const int kHeight = 360;
4605
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004606 video_stream_encoder_->SetSource(&video_source_,
4607 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004608
4609 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4610 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004611 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004612 }
4613
mflodmancc3d4422017-08-03 08:27:51 -07004614 video_stream_encoder_->Stop();
4615 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004616 stats_proxy_.reset();
4617
4618 EXPECT_EQ(0,
4619 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4620}
4621
Per Kjellanderdcef6412020-10-07 15:09:05 +02004622TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4623 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004624 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004625 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004626
4627 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004628 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004629 SimulcastRateAllocator(fake_encoder_.config())
4630 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004631 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004632
Henrik Boström381d1092020-05-12 18:49:07 +02004633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004634 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004635
sprang57c2fff2017-01-16 06:24:02 -08004636 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004637 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4638 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004639 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4640 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4641
Erik Språngd7329ca2019-02-21 21:19:53 +01004642 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004643 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004644 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004645
Per Kjellanderdcef6412020-10-07 15:09:05 +02004646 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004647 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004648 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4649 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004650 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004651 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004652
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004654 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004655 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004656 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004657 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4658 WaitForEncodedFrame(CurrentTimeMs());
4659 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004660 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004661 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004662
mflodmancc3d4422017-08-03 08:27:51 -07004663 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004664}
4665
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004667 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004668 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004669 kVideoLayersAllocation);
4670
4671 const int kDefaultFps = 30;
4672
4673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004674 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004675
4676 video_source_.IncomingCapturedFrame(
4677 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4678 WaitForEncodedFrame(CurrentTimeMs());
4679 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4680 VideoLayersAllocation last_layer_allocation =
4681 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004682 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004683 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4684
4685 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004686 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004687 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004688 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004689 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4690
Erik Språng9d69cbe2020-10-22 17:44:42 +02004691 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004692 int number_of_layers_allocation = 1;
4693 const int64_t start_time_ms = CurrentTimeMs();
4694 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4695 video_source_.IncomingCapturedFrame(
4696 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4697 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004698 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4699 number_of_layers_allocation = sink_.number_of_layers_allocations();
4700 VideoLayersAllocation new_allocation =
4701 sink_.GetLastVideoLayersAllocation();
4702 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4703 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4704 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4705 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4706 .target_bitrate_per_temporal_layer,
4707 last_layer_allocation.active_spatial_layers[0]
4708 .target_bitrate_per_temporal_layer);
4709 last_layer_allocation = new_allocation;
4710 }
4711 }
4712 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4713 video_stream_encoder_->Stop();
4714}
4715
4716TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004717 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721 VideoEncoderConfig video_encoder_config;
4722 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4723 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004724 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725 video_encoder_config.content_type =
4726 VideoEncoderConfig::ContentType::kRealtimeVideo;
4727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004728 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 VideoEncoder::GetDefaultVp8Settings());
4730 for (auto& layer : video_encoder_config.simulcast_layers) {
4731 layer.num_temporal_layers = 2;
4732 }
4733 // Simulcast layers are used for enabling/disabling streams.
4734 video_encoder_config.simulcast_layers[0].active = true;
4735 video_encoder_config.simulcast_layers[1].active = false;
4736 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004737 ConfigureEncoder(std::move(video_encoder_config),
4738 VideoStreamEncoder::BitrateAllocationCallbackType::
4739 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004742 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004743
4744 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4745 WaitForEncodedFrame(CurrentTimeMs());
4746 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4747 VideoLayersAllocation last_layer_allocation =
4748 sink_.GetLastVideoLayersAllocation();
4749
4750 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4751 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4752 .target_bitrate_per_temporal_layer,
4753 SizeIs(2));
4754 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4755 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4756 video_stream_encoder_->Stop();
4757}
4758
4759TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004760 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004761 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4762 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4763 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 VideoEncoderConfig video_encoder_config;
4765 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4766 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004767 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004768 video_encoder_config.content_type =
4769 VideoEncoderConfig::ContentType::kRealtimeVideo;
4770 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004771 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 VideoEncoder::GetDefaultVp8Settings());
4773 for (auto& layer : video_encoder_config.simulcast_layers) {
4774 layer.num_temporal_layers = 2;
4775 }
4776 // Simulcast layers are used for enabling/disabling streams.
4777 video_encoder_config.simulcast_layers[0].active = true;
4778 video_encoder_config.simulcast_layers[1].active = false;
4779 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004780 ConfigureEncoder(std::move(video_encoder_config),
4781 VideoStreamEncoder::BitrateAllocationCallbackType::
4782 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004783
4784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004786
4787 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4788 WaitForEncodedFrame(CurrentTimeMs());
4789 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4790 VideoLayersAllocation last_layer_allocation =
4791 sink_.GetLastVideoLayersAllocation();
4792
4793 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4794 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4795 .target_bitrate_per_temporal_layer,
4796 SizeIs(2));
4797 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4798
4799 video_stream_encoder_->Stop();
4800}
4801
4802TEST_F(VideoStreamEncoderTest,
4803 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4804 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4805 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004806 VideoEncoderConfig video_encoder_config;
4807 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4808 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004809 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004810 video_encoder_config.content_type =
4811 VideoEncoderConfig::ContentType::kRealtimeVideo;
4812 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4813 vp9_settings.numberOfSpatialLayers = 2;
4814 vp9_settings.numberOfTemporalLayers = 2;
4815 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4816 vp9_settings.automaticResizeOn = false;
4817 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004818 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004819 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004820 ConfigureEncoder(std::move(video_encoder_config),
4821 VideoStreamEncoder::BitrateAllocationCallbackType::
4822 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823
4824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004825 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004826
4827 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4828 WaitForEncodedFrame(CurrentTimeMs());
4829 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4830 VideoLayersAllocation last_layer_allocation =
4831 sink_.GetLastVideoLayersAllocation();
4832
4833 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4834 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4835 .target_bitrate_per_temporal_layer,
4836 SizeIs(2));
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4838 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4839 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4840 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4841 .target_bitrate_per_temporal_layer,
4842 SizeIs(2));
4843 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4844 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4845 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4846
4847 // Since full SVC is used, expect the top layer to utilize the full target
4848 // rate.
4849 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4850 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004851 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004852 video_stream_encoder_->Stop();
4853}
4854
4855TEST_F(VideoStreamEncoderTest,
4856 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4857 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4858 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004859 VideoEncoderConfig video_encoder_config;
4860 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4861 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004862 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863 video_encoder_config.content_type =
4864 VideoEncoderConfig::ContentType::kRealtimeVideo;
4865 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4866 vp9_settings.numberOfSpatialLayers = 2;
4867 vp9_settings.numberOfTemporalLayers = 2;
4868 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4869 vp9_settings.automaticResizeOn = false;
4870 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004871 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004872 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004873 ConfigureEncoder(std::move(video_encoder_config),
4874 VideoStreamEncoder::BitrateAllocationCallbackType::
4875 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004876
4877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004878 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004879
4880 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4881 WaitForEncodedFrame(CurrentTimeMs());
4882 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4883 VideoLayersAllocation last_layer_allocation =
4884 sink_.GetLastVideoLayersAllocation();
4885
4886 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4887 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4888 .target_bitrate_per_temporal_layer,
4889 SizeIs(1));
4890 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4891 .target_bitrate_per_temporal_layer,
4892 SizeIs(1));
4893 // Since full SVC is used, expect the top layer to utilize the full target
4894 // rate.
4895 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4896 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004897 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004898 video_stream_encoder_->Stop();
4899}
4900
4901TEST_F(VideoStreamEncoderTest,
4902 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4903 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4904 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004905 VideoEncoderConfig video_encoder_config;
4906 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4907 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004908 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004909 video_encoder_config.content_type =
4910 VideoEncoderConfig::ContentType::kRealtimeVideo;
4911 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4912 vp9_settings.numberOfSpatialLayers = 2;
4913 vp9_settings.numberOfTemporalLayers = 2;
4914 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4915 vp9_settings.automaticResizeOn = false;
4916 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004917 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004918 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004919 ConfigureEncoder(std::move(video_encoder_config),
4920 VideoStreamEncoder::BitrateAllocationCallbackType::
4921 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004922
4923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004924 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004925
4926 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4927 WaitForEncodedFrame(CurrentTimeMs());
4928 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4929 VideoLayersAllocation last_layer_allocation =
4930 sink_.GetLastVideoLayersAllocation();
4931
4932 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4933 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4934 .target_bitrate_per_temporal_layer,
4935 SizeIs(2));
4936 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4937 .target_bitrate_per_temporal_layer,
4938 SizeIs(2));
4939 // Since KSVC is, spatial layers are independend except on key frames.
4940 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4941 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004942 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004943 video_stream_encoder_->Stop();
4944}
4945
4946TEST_F(VideoStreamEncoderTest,
4947 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4948 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4949 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4950 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004951 VideoEncoderConfig video_encoder_config;
4952 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4953 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004954 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004955 video_encoder_config.content_type =
4956 VideoEncoderConfig::ContentType::kRealtimeVideo;
4957 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4958 vp9_settings.numberOfSpatialLayers = 3;
4959 vp9_settings.numberOfTemporalLayers = 2;
4960 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4961 vp9_settings.automaticResizeOn = false;
4962 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004963 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004964 vp9_settings);
4965 // Simulcast layers are used for enabling/disabling streams.
4966 video_encoder_config.simulcast_layers.resize(3);
4967 video_encoder_config.simulcast_layers[0].active = false;
4968 video_encoder_config.simulcast_layers[1].active = true;
4969 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004970 ConfigureEncoder(std::move(video_encoder_config),
4971 VideoStreamEncoder::BitrateAllocationCallbackType::
4972 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004973
4974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004975 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004976
4977 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4978 WaitForEncodedFrame(CurrentTimeMs());
4979 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4980 VideoLayersAllocation last_layer_allocation =
4981 sink_.GetLastVideoLayersAllocation();
4982
4983 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4984 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4985 .target_bitrate_per_temporal_layer,
4986 SizeIs(2));
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4989
4990 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4991 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4992 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4993 .target_bitrate_per_temporal_layer,
4994 SizeIs(2));
4995 // Since full SVC is used, expect the top layer to utilize the full target
4996 // rate.
4997 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4998 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004999 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 video_stream_encoder_->Stop();
5001}
5002
5003TEST_F(VideoStreamEncoderTest,
5004 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5005 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5006 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5007 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005008 VideoEncoderConfig video_encoder_config;
5009 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5010 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005011 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012 video_encoder_config.content_type =
5013 VideoEncoderConfig::ContentType::kRealtimeVideo;
5014 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5015 vp9_settings.numberOfSpatialLayers = 3;
5016 vp9_settings.numberOfTemporalLayers = 2;
5017 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5018 vp9_settings.automaticResizeOn = false;
5019 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005020 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005021 vp9_settings);
5022 // Simulcast layers are used for enabling/disabling streams.
5023 video_encoder_config.simulcast_layers.resize(3);
5024 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005025 ConfigureEncoder(std::move(video_encoder_config),
5026 VideoStreamEncoder::BitrateAllocationCallbackType::
5027 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005028
5029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005030 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005031
5032 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5033 WaitForEncodedFrame(CurrentTimeMs());
5034 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5035 VideoLayersAllocation last_layer_allocation =
5036 sink_.GetLastVideoLayersAllocation();
5037
5038 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5039 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5040 .target_bitrate_per_temporal_layer,
5041 SizeIs(2));
5042 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5043 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5044
5045 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5046 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5047 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5048 .target_bitrate_per_temporal_layer,
5049 SizeIs(2));
5050 video_stream_encoder_->Stop();
5051}
5052
5053TEST_F(VideoStreamEncoderTest,
5054 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5055 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5056 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5057 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005058 VideoEncoderConfig video_encoder_config;
5059 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5060 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005061 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005062 video_encoder_config.content_type =
5063 VideoEncoderConfig::ContentType::kRealtimeVideo;
5064 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5065 vp9_settings.numberOfSpatialLayers = 3;
5066 vp9_settings.numberOfTemporalLayers = 2;
5067 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5068 vp9_settings.automaticResizeOn = false;
5069 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005070 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005071 vp9_settings);
5072 // Simulcast layers are used for enabling/disabling streams.
5073 video_encoder_config.simulcast_layers.resize(3);
5074 video_encoder_config.simulcast_layers[0].active = false;
5075 video_encoder_config.simulcast_layers[1].active = false;
5076 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005077 ConfigureEncoder(std::move(video_encoder_config),
5078 VideoStreamEncoder::BitrateAllocationCallbackType::
5079 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005080
5081 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005082 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005083
5084 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5085 WaitForEncodedFrame(CurrentTimeMs());
5086 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5087 VideoLayersAllocation last_layer_allocation =
5088 sink_.GetLastVideoLayersAllocation();
5089
5090 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5091 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5092 .target_bitrate_per_temporal_layer,
5093 SizeIs(2));
5094 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5095 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5096 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5097 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005098 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005099 video_stream_encoder_->Stop();
5100}
5101
5102TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5103 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005104 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005105 kVideoLayersAllocation);
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005107 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005108
5109 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5110 WaitForEncodedFrame(CurrentTimeMs());
5111 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5112 VideoLayersAllocation last_layer_allocation =
5113 sink_.GetLastVideoLayersAllocation();
5114
5115 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5116 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5117 .target_bitrate_per_temporal_layer,
5118 SizeIs(1));
5119 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5120 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005121 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005122 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5123 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5124 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5125 video_stream_encoder_->Stop();
5126}
5127
5128TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005129 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5130 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005131 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005132 kVideoLayersAllocation);
5133
5134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005135 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005136
5137 video_source_.IncomingCapturedFrame(
5138 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5139 WaitForEncodedFrame(CurrentTimeMs());
5140 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5141 VideoLayersAllocation last_layer_allocation =
5142 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005143 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005144 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5145 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5146 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005147 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005148
5149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005150 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5151 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005152 video_source_.IncomingCapturedFrame(
5153 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5154 WaitForEncodedFrame(CurrentTimeMs());
5155
5156 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5157 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5158 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5159 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5160 .target_bitrate_per_temporal_layer[0],
5161 DataRate::Zero());
5162
5163 video_stream_encoder_->Stop();
5164}
5165
Per Kjellander4190ce92020-12-15 17:24:55 +01005166TEST_F(VideoStreamEncoderTest,
5167 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5168 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005169 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005170 kVideoLayersAllocation);
5171
5172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005173 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5174 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005175
5176 video_source_.IncomingCapturedFrame(
5177 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5178 WaitForEncodedFrame(CurrentTimeMs());
5179 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5180 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5181 SizeIs(2));
5182 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5183 codec_width_);
5184 EXPECT_EQ(
5185 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5186 codec_height_);
5187
5188 video_source_.IncomingCapturedFrame(
5189 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5190 WaitForEncodedFrame(CurrentTimeMs());
5191 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5192 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5193 SizeIs(2));
5194 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5195 codec_width_ / 2);
5196 EXPECT_EQ(
5197 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5198 codec_height_ / 2);
5199
5200 video_stream_encoder_->Stop();
5201}
5202
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005203TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5204 // 2 TLs configured, temporal layers supported by encoder.
5205 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005206 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005207 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005208 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005209 fake_encoder_.SetTemporalLayersSupported(0, true);
5210
5211 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005212 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005213 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005214 kNumTemporalLayers, /*temporal_id*/ 0,
5215 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005216 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005217 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005218 kNumTemporalLayers, /*temporal_id*/ 1,
5219 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005220 VideoBitrateAllocation expected_bitrate;
5221 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5222 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5223
5224 VerifyAllocatedBitrate(expected_bitrate);
5225 video_stream_encoder_->Stop();
5226}
5227
5228TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5229 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005230 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005231 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005232 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005233 fake_encoder_.SetTemporalLayersSupported(0, false);
5234
5235 // Temporal layers not supported by the encoder.
5236 // Total bitrate should be at ti:0.
5237 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005238 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005239
5240 VerifyAllocatedBitrate(expected_bitrate);
5241 video_stream_encoder_->Stop();
5242}
5243
5244TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005245 webrtc::test::ScopedKeyValueConfig field_trials(
5246 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005247 "WebRTC-Video-QualityScalerSettings/"
5248 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5249 // Reset encoder for field trials to take effect.
5250 ConfigureEncoder(video_encoder_config_.Copy());
5251
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005252 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005253 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005254 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005255 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005256 fake_encoder_.SetTemporalLayersSupported(0, true);
5257 fake_encoder_.SetTemporalLayersSupported(1, false);
5258
5259 const int kS0Bps = 150000;
5260 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005261 kS0Bps *
5262 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5263 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005264 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005265 kS0Bps *
5266 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5267 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005268 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005269 // Temporal layers not supported by si:1.
5270 VideoBitrateAllocation expected_bitrate;
5271 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5272 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5273 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5274
5275 VerifyAllocatedBitrate(expected_bitrate);
5276 video_stream_encoder_->Stop();
5277}
5278
Niels Möller7dc26b72017-12-06 10:27:48 +01005279TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5280 const int kFrameWidth = 1280;
5281 const int kFrameHeight = 720;
5282 const int kFramerate = 24;
5283
Henrik Boström381d1092020-05-12 18:49:07 +02005284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005286 test::FrameForwarder source;
5287 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005288 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005289
5290 // Insert a single frame, triggering initial configuration.
5291 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5292 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5293
5294 EXPECT_EQ(
5295 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5296 kDefaultFramerate);
5297
5298 // Trigger reconfigure encoder (without resetting the entire instance).
5299 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005300 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5301 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005302 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005303 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005304 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005305 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5306
5307 // Detector should be updated with fps limit from codec config.
5308 EXPECT_EQ(
5309 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5310 kFramerate);
5311
5312 // Trigger overuse, max framerate should be reduced.
5313 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5314 stats.input_frame_rate = kFramerate;
5315 stats_proxy_->SetMockStats(stats);
5316 video_stream_encoder_->TriggerCpuOveruse();
5317 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5318 int adapted_framerate =
5319 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5320 EXPECT_LT(adapted_framerate, kFramerate);
5321
5322 // Trigger underuse, max framerate should go back to codec configured fps.
5323 // Set extra low fps, to make sure it's actually reset, not just incremented.
5324 stats = stats_proxy_->GetStats();
5325 stats.input_frame_rate = adapted_framerate / 2;
5326 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005327 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5329 EXPECT_EQ(
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5331 kFramerate);
5332
5333 video_stream_encoder_->Stop();
5334}
5335
5336TEST_F(VideoStreamEncoderTest,
5337 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5338 const int kFrameWidth = 1280;
5339 const int kFrameHeight = 720;
5340 const int kLowFramerate = 15;
5341 const int kHighFramerate = 25;
5342
Henrik Boström381d1092020-05-12 18:49:07 +02005343 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005344 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005345 test::FrameForwarder source;
5346 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005347 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005348
5349 // Trigger initial configuration.
5350 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005351 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5352 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005353 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005354 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005355 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005356 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5358
5359 EXPECT_EQ(
5360 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5361 kLowFramerate);
5362
5363 // Trigger overuse, max framerate should be reduced.
5364 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5365 stats.input_frame_rate = kLowFramerate;
5366 stats_proxy_->SetMockStats(stats);
5367 video_stream_encoder_->TriggerCpuOveruse();
5368 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5369 int adapted_framerate =
5370 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5371 EXPECT_LT(adapted_framerate, kLowFramerate);
5372
5373 // Reconfigure the encoder with a new (higher max framerate), max fps should
5374 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005375 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005376 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5377 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005378 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005379 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5380
5381 EXPECT_EQ(
5382 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5383 adapted_framerate);
5384
5385 // Trigger underuse, max framerate should go back to codec configured fps.
5386 stats = stats_proxy_->GetStats();
5387 stats.input_frame_rate = adapted_framerate;
5388 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005389 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5391 EXPECT_EQ(
5392 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5393 kHighFramerate);
5394
5395 video_stream_encoder_->Stop();
5396}
5397
mflodmancc3d4422017-08-03 08:27:51 -07005398TEST_F(VideoStreamEncoderTest,
5399 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005400 const int kFrameWidth = 1280;
5401 const int kFrameHeight = 720;
5402 const int kFramerate = 24;
5403
Henrik Boström381d1092020-05-12 18:49:07 +02005404 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005405 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005406 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005407 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005408 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005409
5410 // Trigger initial configuration.
5411 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005412 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5413 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005414 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005415 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005416 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005417 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005419
Niels Möller7dc26b72017-12-06 10:27:48 +01005420 EXPECT_EQ(
5421 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5422 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005423
5424 // Trigger overuse, max framerate should be reduced.
5425 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5426 stats.input_frame_rate = kFramerate;
5427 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005428 video_stream_encoder_->TriggerCpuOveruse();
5429 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005430 int adapted_framerate =
5431 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005432 EXPECT_LT(adapted_framerate, kFramerate);
5433
5434 // Change degradation preference to not enable framerate scaling. Target
5435 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005436 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005437 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005438 EXPECT_EQ(
5439 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5440 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005441
mflodmancc3d4422017-08-03 08:27:51 -07005442 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005443}
5444
mflodmancc3d4422017-08-03 08:27:51 -07005445TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005446 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005448 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5449 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5450 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005451 const int kWidth = 640;
5452 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005453
asaperssonfab67072017-04-04 05:51:49 -07005454 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005455
5456 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005457 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005458
5459 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005460 EXPECT_TRUE_WAIT(
5461 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005462
sprangc5d62e22017-04-02 23:53:04 -07005463 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005464
asaperssonfab67072017-04-04 05:51:49 -07005465 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005466 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005467 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005468
5469 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005470 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005471
Henrik Boström2671dac2020-05-19 16:29:09 +02005472 EXPECT_TRUE_WAIT(
5473 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005474
mflodmancc3d4422017-08-03 08:27:51 -07005475 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005476}
5477
mflodmancc3d4422017-08-03 08:27:51 -07005478TEST_F(VideoStreamEncoderTest,
5479 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005480 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005482 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5483 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5484 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005485 const int kWidth = 640;
5486 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005487
5488 // We expect the n initial frames to get dropped.
5489 int i;
5490 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005491 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005492 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005493 }
5494 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005495 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005496 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005497
5498 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005499 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005500
mflodmancc3d4422017-08-03 08:27:51 -07005501 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005502}
5503
mflodmancc3d4422017-08-03 08:27:51 -07005504TEST_F(VideoStreamEncoderTest,
5505 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005506 const int kWidth = 640;
5507 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005509 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005510
5511 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005512 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005513 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005514
asaperssonfab67072017-04-04 05:51:49 -07005515 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005516 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005517 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005518
mflodmancc3d4422017-08-03 08:27:51 -07005519 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005520}
5521
mflodmancc3d4422017-08-03 08:27:51 -07005522TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005523 const int kWidth = 640;
5524 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005525 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005526
5527 VideoEncoderConfig video_encoder_config;
5528 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5529 // Make format different, to force recreation of encoder.
5530 video_encoder_config.video_format.parameters["foo"] = "foo";
5531 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005532 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005534 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005535
kthelgasonb83797b2017-02-14 11:57:25 -08005536 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005537 video_stream_encoder_->SetSource(&video_source_,
5538 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005539
asaperssonfab67072017-04-04 05:51:49 -07005540 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005541 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005542 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005543
mflodmancc3d4422017-08-03 08:27:51 -07005544 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005545 fake_encoder_.SetQualityScaling(true);
5546}
5547
Åsa Persson139f4dc2019-08-02 09:29:58 +02005548TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005549 webrtc::test::ScopedKeyValueConfig field_trials(
5550 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005551 "WebRTC-Video-QualityScalerSettings/"
5552 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5553 // Reset encoder for field trials to take effect.
5554 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005555 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5556 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005557 const int kWidth = 640;
5558 const int kHeight = 360;
5559
Henrik Boström381d1092020-05-12 18:49:07 +02005560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005561 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005562 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5563 // Frame should not be dropped.
5564 WaitForEncodedFrame(1);
5565
Henrik Boström381d1092020-05-12 18:49:07 +02005566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005567 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5568 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5569 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005570 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5571 // Frame should not be dropped.
5572 WaitForEncodedFrame(2);
5573
Henrik Boström381d1092020-05-12 18:49:07 +02005574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005575 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5576 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5577 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005578 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5579 // Expect to drop this frame, the wait should time out.
5580 ExpectDroppedFrame();
5581
5582 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005583 EXPECT_TRUE_WAIT(
5584 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005585 video_stream_encoder_->Stop();
5586}
5587
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005588TEST_F(VideoStreamEncoderTest,
5589 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005590 webrtc::test::ScopedKeyValueConfig field_trials(
5591 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005592 "WebRTC-Video-QualityScalerSettings/"
5593 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5594 fake_encoder_.SetQualityScaling(false);
5595 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005596 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5597 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005598 const int kWidth = 640;
5599 const int kHeight = 360;
5600
5601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005603 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5604 // Frame should not be dropped.
5605 WaitForEncodedFrame(1);
5606
5607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5608 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5609 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5610 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5611 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5612 // Frame should not be dropped.
5613 WaitForEncodedFrame(2);
5614
5615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5616 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5617 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5618 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5619 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5620 // Not dropped since quality scaling is disabled.
5621 WaitForEncodedFrame(3);
5622
5623 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005624 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005625 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5626
5627 video_stream_encoder_->Stop();
5628}
5629
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005630TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005631 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005632 // Set simulcast.
5633 ResetEncoder("VP8", 3, 1, 1, false);
5634 fake_encoder_.SetQualityScaling(true);
5635 const int kWidth = 1280;
5636 const int kHeight = 720;
5637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005638 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005639 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5640 // Frame should not be dropped.
5641 WaitForEncodedFrame(1);
5642
5643 // Trigger QVGA "singlecast"
5644 // Update the config.
5645 VideoEncoderConfig video_encoder_config;
5646 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5647 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005648 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005649 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005650 "VP8", /*max qp*/ 56, /*screencast*/ false,
5651 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005652 for (auto& layer : video_encoder_config.simulcast_layers) {
5653 layer.num_temporal_layers = 1;
5654 layer.max_framerate = kDefaultFramerate;
5655 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005656 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005657 video_encoder_config.content_type =
5658 VideoEncoderConfig::ContentType::kRealtimeVideo;
5659
5660 video_encoder_config.simulcast_layers[0].active = true;
5661 video_encoder_config.simulcast_layers[1].active = false;
5662 video_encoder_config.simulcast_layers[2].active = false;
5663
5664 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5665 kMaxPayloadLength);
5666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5667
5668 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5669 // Frame should not be dropped.
5670 WaitForEncodedFrame(2);
5671
5672 // Trigger HD "singlecast"
5673 video_encoder_config.simulcast_layers[0].active = false;
5674 video_encoder_config.simulcast_layers[1].active = false;
5675 video_encoder_config.simulcast_layers[2].active = true;
5676
5677 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5678 kMaxPayloadLength);
5679 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5680
5681 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5682 // Frame should be dropped because of initial frame drop.
5683 ExpectDroppedFrame();
5684
5685 // Expect the sink_wants to specify a scaled frame.
5686 EXPECT_TRUE_WAIT(
5687 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5688 video_stream_encoder_->Stop();
5689}
5690
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005691TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005692 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005693 // Set simulcast.
5694 ResetEncoder("VP9", 1, 1, 3, false);
5695 fake_encoder_.SetQualityScaling(true);
5696 const int kWidth = 1280;
5697 const int kHeight = 720;
5698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005699 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005700 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5701 // Frame should not be dropped.
5702 WaitForEncodedFrame(1);
5703
5704 // Trigger QVGA "singlecast"
5705 // Update the config.
5706 VideoEncoderConfig video_encoder_config;
5707 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5708 &video_encoder_config);
5709 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5710 vp9_settings.numberOfSpatialLayers = 3;
5711 // Since only one layer is active - automatic resize should be enabled.
5712 vp9_settings.automaticResizeOn = true;
5713 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005714 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005715 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005716 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005717 video_encoder_config.content_type =
5718 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005719 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005720 // which SVC layers are active.
5721 video_encoder_config.simulcast_layers.resize(3);
5722
5723 video_encoder_config.simulcast_layers[0].active = true;
5724 video_encoder_config.simulcast_layers[1].active = false;
5725 video_encoder_config.simulcast_layers[2].active = false;
5726
5727 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5728 kMaxPayloadLength);
5729 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5730
5731 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5732 // Frame should not be dropped.
5733 WaitForEncodedFrame(2);
5734
5735 // Trigger HD "singlecast"
5736 video_encoder_config.simulcast_layers[0].active = false;
5737 video_encoder_config.simulcast_layers[1].active = false;
5738 video_encoder_config.simulcast_layers[2].active = true;
5739
5740 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5741 kMaxPayloadLength);
5742 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5743
5744 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5745 // Frame should be dropped because of initial frame drop.
5746 ExpectDroppedFrame();
5747
5748 // Expect the sink_wants to specify a scaled frame.
5749 EXPECT_TRUE_WAIT(
5750 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5751 video_stream_encoder_->Stop();
5752}
5753
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005754TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005755 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5756 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5757 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5758 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5759 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5760 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5761 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5762 fake_encoder_.SetResolutionBitrateLimits(
5763 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5764
5765 VideoEncoderConfig video_encoder_config;
5766 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5767 &video_encoder_config);
5768 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5769 vp9_settings.numberOfSpatialLayers = 3;
5770 // Since only one layer is active - automatic resize should be enabled.
5771 vp9_settings.automaticResizeOn = true;
5772 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005773 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005774 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005775 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005776 video_encoder_config.content_type =
5777 VideoEncoderConfig::ContentType::kRealtimeVideo;
5778 // Simulcast layers are used to indicate which spatial layers are active.
5779 video_encoder_config.simulcast_layers.resize(3);
5780 video_encoder_config.simulcast_layers[0].active = false;
5781 video_encoder_config.simulcast_layers[1].active = true;
5782 video_encoder_config.simulcast_layers[2].active = false;
5783
5784 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5785 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005786
5787 // The encoder bitrate limits for 360p should be used.
5788 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005789 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005790 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5791 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5792 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5793 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5794 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5795 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005796 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005797 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005798 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005799 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005800
5801 // The encoder bitrate limits for 270p should be used.
5802 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005803 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005804 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5805 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5806 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5807 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5808 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5809 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005810 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005811 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005812 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005813 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005814
5815 video_stream_encoder_->Stop();
5816}
5817
5818TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005819 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5820 VideoEncoderConfig video_encoder_config;
5821 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5822 &video_encoder_config);
5823 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5824 vp9_settings.numberOfSpatialLayers = 3;
5825 // Since only one layer is active - automatic resize should be enabled.
5826 vp9_settings.automaticResizeOn = true;
5827 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005828 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005829 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005830 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005831 video_encoder_config.content_type =
5832 VideoEncoderConfig::ContentType::kRealtimeVideo;
5833 // Simulcast layers are used to indicate which spatial layers are active.
5834 video_encoder_config.simulcast_layers.resize(3);
5835 video_encoder_config.simulcast_layers[0].active = false;
5836 video_encoder_config.simulcast_layers[1].active = true;
5837 video_encoder_config.simulcast_layers[2].active = false;
5838
5839 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5840 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005841
5842 // The default bitrate limits for 360p should be used.
5843 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005844 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5845 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005846 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005848 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5849 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5850 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5851 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5852 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5853 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005854 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005855 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005856 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005857 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005858
5859 // The default bitrate limits for 270p should be used.
5860 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005861 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5862 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005863 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005864 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005865 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5866 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5867 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5868 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5869 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5870 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005871 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005872 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005873 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005874 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005875
5876 video_stream_encoder_->Stop();
5877}
5878
5879TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005880 webrtc::test::ScopedKeyValueConfig field_trials(
5881 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005882 VideoEncoderConfig video_encoder_config;
5883 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5884 &video_encoder_config);
5885 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5886 vp9_settings.numberOfSpatialLayers = 3;
5887 // Since only one layer is active - automatic resize should be enabled.
5888 vp9_settings.automaticResizeOn = true;
5889 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005890 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005891 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005892 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005893 video_encoder_config.content_type =
5894 VideoEncoderConfig::ContentType::kRealtimeVideo;
5895 // Simulcast layers are used to indicate which spatial layers are active.
5896 video_encoder_config.simulcast_layers.resize(3);
5897 video_encoder_config.simulcast_layers[0].active = false;
5898 video_encoder_config.simulcast_layers[1].active = true;
5899 video_encoder_config.simulcast_layers[2].active = false;
5900
5901 // Reset encoder for field trials to take effect.
5902 ConfigureEncoder(video_encoder_config.Copy());
5903
5904 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5905 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005906
5907 // The default bitrate limits for 360p should not be used.
5908 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005909 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5910 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005911 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005912 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005913 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5914 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5915 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5916 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5917 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5918 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005919 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005920 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005921
5922 video_stream_encoder_->Stop();
5923}
5924
5925TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5926 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5927 /*num_spatial_layers=*/1, /*screenshare=*/false);
5928
5929 // The default singlecast bitrate limits for 720p should not be used.
5930 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005931 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5932 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005933 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005935 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5936 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5937 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5938 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5939 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5940 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005941 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005942 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005943
5944 video_stream_encoder_->Stop();
5945}
5946
5947TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005948 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5949 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5950 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5951 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5952 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5953 fake_encoder_.SetResolutionBitrateLimits(
5954 {kEncoderLimits180p, kEncoderLimits720p});
5955
5956 VideoEncoderConfig video_encoder_config;
5957 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5958 &video_encoder_config);
5959 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5960 vp9_settings.numberOfSpatialLayers = 3;
5961 // Since only one layer is active - automatic resize should be enabled.
5962 vp9_settings.automaticResizeOn = true;
5963 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005964 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005965 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005966 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005967 video_encoder_config.content_type =
5968 VideoEncoderConfig::ContentType::kRealtimeVideo;
5969 // Simulcast layers are used to indicate which spatial layers are active.
5970 video_encoder_config.simulcast_layers.resize(3);
5971 video_encoder_config.simulcast_layers[0].active = true;
5972 video_encoder_config.simulcast_layers[1].active = false;
5973 video_encoder_config.simulcast_layers[2].active = false;
5974
5975 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5976 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005977
5978 // Limits not applied on lowest stream, limits for 180p should not be used.
5979 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005980 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005981 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5982 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5983 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5984 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5985 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5986 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005987 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005988 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005989 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005990 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005991
5992 video_stream_encoder_->Stop();
5993}
5994
5995TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005996 InitialFrameDropActivatesWhenResolutionIncreases) {
5997 const int kWidth = 640;
5998 const int kHeight = 360;
5999
6000 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006001 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006002 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6003 // Frame should not be dropped.
6004 WaitForEncodedFrame(1);
6005
6006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006007 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006008 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6009 // Frame should not be dropped, bitrate not too low for frame.
6010 WaitForEncodedFrame(2);
6011
6012 // Incoming resolution increases.
6013 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6014 // Expect to drop this frame, bitrate too low for frame.
6015 ExpectDroppedFrame();
6016
6017 // Expect the sink_wants to specify a scaled frame.
6018 EXPECT_TRUE_WAIT(
6019 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6020 video_stream_encoder_->Stop();
6021}
6022
6023TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6024 const int kWidth = 640;
6025 const int kHeight = 360;
6026 // So that quality scaling doesn't happen by itself.
6027 fake_encoder_.SetQp(kQpHigh);
6028
6029 AdaptingFrameForwarder source(&time_controller_);
6030 source.set_adaptation_enabled(true);
6031 video_stream_encoder_->SetSource(
6032 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6033
6034 int timestamp = 1;
6035
6036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006037 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006038 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6039 WaitForEncodedFrame(timestamp);
6040 timestamp += 9000;
6041 // Long pause to disable all first BWE drop logic.
6042 AdvanceTime(TimeDelta::Millis(1000));
6043
6044 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006045 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006046 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6047 // Not dropped frame, as initial frame drop is disabled by now.
6048 WaitForEncodedFrame(timestamp);
6049 timestamp += 9000;
6050 AdvanceTime(TimeDelta::Millis(100));
6051
6052 // Quality adaptation down.
6053 video_stream_encoder_->TriggerQualityLow();
6054
6055 // Adaptation has an effect.
6056 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6057 5000);
6058
6059 // Frame isn't dropped as initial frame dropper is disabled.
6060 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6061 WaitForEncodedFrame(timestamp);
6062 timestamp += 9000;
6063 AdvanceTime(TimeDelta::Millis(100));
6064
6065 // Quality adaptation up.
6066 video_stream_encoder_->TriggerQualityHigh();
6067
6068 // Adaptation has an effect.
6069 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6070 5000);
6071
6072 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6073 // Frame should not be dropped, as initial framedropper is off.
6074 WaitForEncodedFrame(timestamp);
6075
6076 video_stream_encoder_->Stop();
6077}
6078
Åsa Persson7f354f82021-02-04 15:52:15 +01006079TEST_F(VideoStreamEncoderTest,
6080 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6081 const int kMinStartBps360p = 222000;
6082 fake_encoder_.SetResolutionBitrateLimits(
6083 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6084 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6085 800000)});
6086
6087 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6088 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6089 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6091 0, 0, 0);
6092 // Frame should not be dropped, bitrate not too low for frame.
6093 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6094 WaitForEncodedFrame(1);
6095
6096 // Incoming resolution increases, initial frame drop activates.
6097 // Frame should be dropped, link allocation too low for frame.
6098 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6099 ExpectDroppedFrame();
6100
6101 // Expect sink_wants to specify a scaled frame.
6102 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6103 5000);
6104 video_stream_encoder_->Stop();
6105}
6106
6107TEST_F(VideoStreamEncoderTest,
6108 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6109 const int kMinStartBps360p = 222000;
6110 fake_encoder_.SetResolutionBitrateLimits(
6111 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6112 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6113 800000)});
6114
6115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6116 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6117 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6118 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6119 0, 0, 0);
6120 // Frame should not be dropped, bitrate not too low for frame.
6121 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6122 WaitForEncodedFrame(1);
6123
6124 // Incoming resolution increases, initial frame drop activates.
6125 // Frame should be dropped, link allocation not too low for frame.
6126 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6127 WaitForEncodedFrame(2);
6128
6129 video_stream_encoder_->Stop();
6130}
6131
Åsa Perssone644a032019-11-08 15:56:00 +01006132TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006133 webrtc::test::ScopedKeyValueConfig field_trials(
6134 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006135 "WebRTC-Video-QualityRampupSettings/"
6136 "min_pixels:921600,min_duration_ms:2000/");
6137
6138 const int kWidth = 1280;
6139 const int kHeight = 720;
6140 const int kFps = 10;
6141 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006142
6143 // Reset encoder for field trials to take effect.
6144 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006145 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006146 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006147 ConfigureEncoder(std::move(config));
6148 fake_encoder_.SetQp(kQpLow);
6149
6150 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006151 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006152 source.set_adaptation_enabled(true);
6153 video_stream_encoder_->SetSource(&source,
6154 DegradationPreference::MAINTAIN_FRAMERATE);
6155
6156 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006157 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006158 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006159 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006160
6161 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006162 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006163 int64_t timestamp_ms = kFrameIntervalMs;
6164 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6165 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006166 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6167 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006168
6169 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6171 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006172
Artem Titovab30d722021-07-27 16:22:11 +02006173 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006174 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006175 for (size_t i = 1; i <= 10; i++) {
6176 timestamp_ms += kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 WaitForEncodedFrame(timestamp_ms);
6179 }
Åsa Persson06defc42021-09-10 15:28:48 +02006180
6181 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6182 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6183 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6184 timestamp_ms += kFrameIntervalMs;
6185 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6186 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006187 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6188 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6189
Åsa Persson06defc42021-09-10 15:28:48 +02006190 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006194 // The ramp-up code involves the adaptation queue, give it time to execute.
6195 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006196 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006197 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006198
6199 // Frame should not be adapted.
6200 timestamp_ms += kFrameIntervalMs;
6201 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6202 WaitForEncodedFrame(kWidth, kHeight);
6203 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6204
6205 video_stream_encoder_->Stop();
6206}
6207
mflodmancc3d4422017-08-03 08:27:51 -07006208TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006209 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006210 webrtc::test::ScopedKeyValueConfig field_trials(
6211 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006212 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006213 source.set_adaptation_enabled(true);
6214 video_stream_encoder_->SetSource(&source,
6215 DegradationPreference::MAINTAIN_FRAMERATE);
6216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006217 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006218 fake_encoder_.SetQp(kQpHigh + 1);
6219 const int kWidth = 1280;
6220 const int kHeight = 720;
6221 const int64_t kFrameIntervalMs = 100;
6222 int64_t timestamp_ms = kFrameIntervalMs;
6223 for (size_t i = 1; i <= 100; i++) {
6224 timestamp_ms += kFrameIntervalMs;
6225 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6226 WaitForEncodedFrame(timestamp_ms);
6227 }
6228 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6229 // for the first time.
6230 // TODO(eshr): We should avoid these waits by using threads with simulated
6231 // time.
6232 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6233 2000 * 2.5 * 2);
6234 timestamp_ms += kFrameIntervalMs;
6235 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6236 WaitForEncodedFrame(timestamp_ms);
6237 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6238 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6239 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6240
6241 // Disable Quality scaling by turning off scaler on the encoder and
6242 // reconfiguring.
6243 fake_encoder_.SetQualityScaling(false);
6244 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6245 kMaxPayloadLength);
6246 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006247 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006248 // Since we turned off the quality scaler, the adaptations made by it are
6249 // removed.
6250 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6252
6253 video_stream_encoder_->Stop();
6254}
6255
6256TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006257 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6258 const int kTooSmallWidth = 10;
6259 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006261 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006262
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006263 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006264 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006265 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006266 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006267 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006268 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6269
6270 // Trigger adapt down, too small frame, expect no change.
6271 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006272 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006273 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006274 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006275 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6276 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6277
mflodmancc3d4422017-08-03 08:27:51 -07006278 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006279}
6280
mflodmancc3d4422017-08-03 08:27:51 -07006281TEST_F(VideoStreamEncoderTest,
6282 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006283 const int kTooSmallWidth = 10;
6284 const int kTooSmallHeight = 10;
6285 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006288
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006289 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006290 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006291 video_stream_encoder_->SetSource(&source,
6292 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006293 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6295 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6296
6297 // Trigger adapt down, expect limited framerate.
6298 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006299 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006300 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006301 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006302 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6303 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6304 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6305
6306 // Trigger adapt down, too small frame, expect no change.
6307 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006308 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006309 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006310 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006311 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6312 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6313 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6314
mflodmancc3d4422017-08-03 08:27:51 -07006315 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006316}
6317
mflodmancc3d4422017-08-03 08:27:51 -07006318TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006319 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006321 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006322 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006323 const int kFrameWidth = 1280;
6324 const int kFrameHeight = 720;
6325 video_source_.IncomingCapturedFrame(
6326 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006327 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006328 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006329}
6330
sprangb1ca0732017-02-01 08:38:12 -08006331// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006332TEST_F(VideoStreamEncoderTest,
6333 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006334 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006335 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006336
6337 const int kFrameWidth = 1280;
6338 const int kFrameHeight = 720;
6339 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006340 // requested by
6341 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006342 video_source_.set_adaptation_enabled(true);
6343
6344 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006345 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006346 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006347
6348 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006349 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006350 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006351 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006352 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006353
asaperssonfab67072017-04-04 05:51:49 -07006354 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006355 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006356 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006357 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006358 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006359
mflodmancc3d4422017-08-03 08:27:51 -07006360 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006361}
sprangfe627f32017-03-29 08:24:59 -07006362
mflodmancc3d4422017-08-03 08:27:51 -07006363TEST_F(VideoStreamEncoderTest,
6364 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006365 const int kFrameWidth = 1280;
6366 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006367
Henrik Boström381d1092020-05-12 18:49:07 +02006368 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006369 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006370 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006371 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006372 video_source_.set_adaptation_enabled(true);
6373
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006374 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006375
6376 video_source_.IncomingCapturedFrame(
6377 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006378 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006379
6380 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006381 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006382
6383 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006384 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006385 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006386 video_source_.IncomingCapturedFrame(
6387 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006388 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006389 }
6390
6391 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006392 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006393 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006394 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006395 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006396 video_source_.IncomingCapturedFrame(
6397 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006398 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006399 ++num_frames_dropped;
6400 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006401 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006402 }
6403 }
6404
sprang4847ae62017-06-27 07:06:52 -07006405 // Add some slack to account for frames dropped by the frame dropper.
6406 const int kErrorMargin = 1;
6407 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006408 kErrorMargin);
6409
6410 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006411 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006412 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006413 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006414 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006415 video_source_.IncomingCapturedFrame(
6416 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006417 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006418 ++num_frames_dropped;
6419 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006420 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006421 }
6422 }
sprang4847ae62017-06-27 07:06:52 -07006423 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006424 kErrorMargin);
6425
6426 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006427 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006428 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006429 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006430 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006431 video_source_.IncomingCapturedFrame(
6432 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006433 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006434 ++num_frames_dropped;
6435 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006436 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006437 }
6438 }
sprang4847ae62017-06-27 07:06:52 -07006439 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006440 kErrorMargin);
6441
6442 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006443 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006444 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006445 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006446 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006447 video_source_.IncomingCapturedFrame(
6448 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006449 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006450 ++num_frames_dropped;
6451 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006452 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006453 }
6454 }
6455 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6456
mflodmancc3d4422017-08-03 08:27:51 -07006457 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006458}
6459
mflodmancc3d4422017-08-03 08:27:51 -07006460TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006461 const int kFramerateFps = 5;
6462 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006463 const int kFrameWidth = 1280;
6464 const int kFrameHeight = 720;
6465
sprang4847ae62017-06-27 07:06:52 -07006466 // Reconfigure encoder with two temporal layers and screensharing, which will
6467 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006468 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006469
Henrik Boström381d1092020-05-12 18:49:07 +02006470 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006471 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006472 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006473 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006474 video_source_.set_adaptation_enabled(true);
6475
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006476 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006477
6478 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006479 rtc::VideoSinkWants last_wants;
6480 do {
6481 last_wants = video_source_.sink_wants();
6482
sprangc5d62e22017-04-02 23:53:04 -07006483 // Insert frames to get a new fps estimate...
6484 for (int j = 0; j < kFramerateFps; ++j) {
6485 video_source_.IncomingCapturedFrame(
6486 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006487 if (video_source_.last_sent_width()) {
6488 sink_.WaitForEncodedFrame(timestamp_ms);
6489 }
sprangc5d62e22017-04-02 23:53:04 -07006490 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006491 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006492 }
6493 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006494 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006495 } while (video_source_.sink_wants().max_framerate_fps <
6496 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006497
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006498 EXPECT_THAT(video_source_.sink_wants(),
6499 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006500
mflodmancc3d4422017-08-03 08:27:51 -07006501 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006502}
asaperssonf7e294d2017-06-13 23:25:22 -07006503
mflodmancc3d4422017-08-03 08:27:51 -07006504TEST_F(VideoStreamEncoderTest,
6505 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006506 const int kWidth = 1280;
6507 const int kHeight = 720;
6508 const int64_t kFrameIntervalMs = 150;
6509 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006511 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006512
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006513 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006514 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006515 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006516 video_stream_encoder_->SetSource(&source,
6517 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006518 timestamp_ms += kFrameIntervalMs;
6519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006520 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006521 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006522 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6524 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6525
6526 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006527 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006528 timestamp_ms += kFrameIntervalMs;
6529 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006530 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006531 EXPECT_THAT(source.sink_wants(),
6532 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006533 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6535 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6536
6537 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006538 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006539 timestamp_ms += kFrameIntervalMs;
6540 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006541 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006542 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6544 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6545 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6546
6547 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006548 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006549 timestamp_ms += kFrameIntervalMs;
6550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006551 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006552 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006553 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6555 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6556
6557 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006558 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006559 timestamp_ms += kFrameIntervalMs;
6560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006561 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006562 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006563 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6565 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6566
6567 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006568 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006569 timestamp_ms += kFrameIntervalMs;
6570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006571 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006572 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006573 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6575 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6576
6577 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006578 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006579 timestamp_ms += kFrameIntervalMs;
6580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006581 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006582 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6585 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6586
6587 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006588 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006589 timestamp_ms += kFrameIntervalMs;
6590 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006591 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006592 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006593 rtc::VideoSinkWants last_wants = source.sink_wants();
6594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6596 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6597
6598 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006599 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006600 timestamp_ms += kFrameIntervalMs;
6601 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006602 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006603 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6606 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6607
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006608 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006609 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006610 timestamp_ms += kFrameIntervalMs;
6611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006612 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006613 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6616 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6617
6618 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006619 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006620 timestamp_ms += kFrameIntervalMs;
6621 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006622 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006623 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006624 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6625 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6626 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6627
6628 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006629 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006630 timestamp_ms += kFrameIntervalMs;
6631 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006632 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006633 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6636 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6637
6638 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006639 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006640 timestamp_ms += kFrameIntervalMs;
6641 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006642 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006643 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006644 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6646 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6647
6648 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006649 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006650 timestamp_ms += kFrameIntervalMs;
6651 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006652 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006653 EXPECT_THAT(source.sink_wants(), FpsMax());
6654 EXPECT_EQ(source.sink_wants().max_pixel_count,
6655 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6658 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6659
6660 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006661 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006662 timestamp_ms += kFrameIntervalMs;
6663 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006664 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006665 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006666 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6667 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6668 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6669
Åsa Persson30ab0152019-08-27 12:22:33 +02006670 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006671 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006672 timestamp_ms += kFrameIntervalMs;
6673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006674 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006676 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6679 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6680
6681 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006682 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006683 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006684 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6685
mflodmancc3d4422017-08-03 08:27:51 -07006686 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006687}
6688
mflodmancc3d4422017-08-03 08:27:51 -07006689TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006690 const int kWidth = 1280;
6691 const int kHeight = 720;
6692 const int64_t kFrameIntervalMs = 150;
6693 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006695 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006696
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006697 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006698 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006699 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006700 video_stream_encoder_->SetSource(&source,
6701 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006702 timestamp_ms += kFrameIntervalMs;
6703 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006704 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006705 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006706 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6707 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6708 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6709 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6710 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6711 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6712
6713 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006714 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006715 timestamp_ms += kFrameIntervalMs;
6716 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006717 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006718 EXPECT_THAT(source.sink_wants(),
6719 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006720 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6721 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6722 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6724 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6725 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6726
6727 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006728 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006729 timestamp_ms += kFrameIntervalMs;
6730 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006731 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006733 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6734 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6735 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6737 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6739
6740 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006741 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006742 timestamp_ms += kFrameIntervalMs;
6743 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006744 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006745 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006746 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006747 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6748 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6750 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6751 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6752
Evan Shrubsole64469032020-06-11 10:45:29 +02006753 // Trigger cpu adapt up, expect no change since QP is most limited.
6754 {
6755 // Store current sink wants since we expect no change and if there is no
6756 // change then last_wants() is not updated.
6757 auto previous_sink_wants = source.sink_wants();
6758 video_stream_encoder_->TriggerCpuUnderuse();
6759 timestamp_ms += kFrameIntervalMs;
6760 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6761 WaitForEncodedFrame(timestamp_ms);
6762 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6763 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6764 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6765 }
6766
6767 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6768 video_stream_encoder_->TriggerQualityHigh();
6769 timestamp_ms += kFrameIntervalMs;
6770 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6771 WaitForEncodedFrame(timestamp_ms);
6772 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6773 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6774 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6775 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6776 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6777 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6778 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6779
6780 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6781 // expect increased resolution (960x540@30fps).
6782 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006783 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006784 timestamp_ms += kFrameIntervalMs;
6785 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006786 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006787 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006788 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6789 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6790 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6792 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006793 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006794
Evan Shrubsole64469032020-06-11 10:45:29 +02006795 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6796 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006797 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006798 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006799 timestamp_ms += kFrameIntervalMs;
6800 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006801 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006802 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006803 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006804 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6806 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6807 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6808 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006809 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006810
6811 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006812 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006813 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006814 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006815 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006816
mflodmancc3d4422017-08-03 08:27:51 -07006817 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006818}
6819
mflodmancc3d4422017-08-03 08:27:51 -07006820TEST_F(VideoStreamEncoderTest,
6821 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006822 const int kWidth = 640;
6823 const int kHeight = 360;
6824 const int kFpsLimit = 15;
6825 const int64_t kFrameIntervalMs = 150;
6826 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006828 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006829
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006830 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006831 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006832 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006833 video_stream_encoder_->SetSource(&source,
6834 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006835 timestamp_ms += kFrameIntervalMs;
6836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006837 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006838 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006839 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6841 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6843 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6845
6846 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006847 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006848 timestamp_ms += kFrameIntervalMs;
6849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006850 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006851 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6855 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6856 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6857 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6858
6859 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006860 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006861 timestamp_ms += kFrameIntervalMs;
6862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006863 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006864 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006865 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006866 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6869 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6870 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6871
Evan Shrubsole64469032020-06-11 10:45:29 +02006872 // Trigger cpu adapt up, expect no change because quality is most limited.
6873 {
6874 auto previous_sink_wants = source.sink_wants();
6875 // Store current sink wants since we expect no change ind if there is no
6876 // change then last__wants() is not updated.
6877 video_stream_encoder_->TriggerCpuUnderuse();
6878 timestamp_ms += kFrameIntervalMs;
6879 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6880 WaitForEncodedFrame(timestamp_ms);
6881 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6882 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6884 }
6885
6886 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6887 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006888 timestamp_ms += kFrameIntervalMs;
6889 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006890 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006891 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006892 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6893 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6894 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006895 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6896 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6897 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006898
Evan Shrubsole64469032020-06-11 10:45:29 +02006899 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006900 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006901 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006902 timestamp_ms += kFrameIntervalMs;
6903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006904 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006905 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006906 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6907 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6908 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6909 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6910 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006911 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006912
6913 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006914 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006915 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006916 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006917 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006918
mflodmancc3d4422017-08-03 08:27:51 -07006919 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006920}
6921
mflodmancc3d4422017-08-03 08:27:51 -07006922TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006923 const int kFrameWidth = 1920;
6924 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006925 // 2/3 of 1920.
6926 const int kAdaptedFrameWidth = 1280;
6927 // 2/3 of 1080.
6928 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006929 const int kFramerate = 24;
6930
Henrik Boström381d1092020-05-12 18:49:07 +02006931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006933 // Trigger reconfigure encoder (without resetting the entire instance).
6934 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006935 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6936 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006937 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006938 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006939 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006940 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006941 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006942 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006943
6944 video_source_.set_adaptation_enabled(true);
6945
6946 video_source_.IncomingCapturedFrame(
6947 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006948 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006949
6950 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006951 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006952 video_source_.IncomingCapturedFrame(
6953 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006954 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006955
mflodmancc3d4422017-08-03 08:27:51 -07006956 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006957}
6958
mflodmancc3d4422017-08-03 08:27:51 -07006959TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006960 const int kFrameWidth = 1280;
6961 const int kFrameHeight = 720;
6962 const int kLowFps = 2;
6963 const int kHighFps = 30;
6964
Henrik Boström381d1092020-05-12 18:49:07 +02006965 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006966 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006967
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006968 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006969 max_framerate_ = kLowFps;
6970
6971 // Insert 2 seconds of 2fps video.
6972 for (int i = 0; i < kLowFps * 2; ++i) {
6973 video_source_.IncomingCapturedFrame(
6974 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6975 WaitForEncodedFrame(timestamp_ms);
6976 timestamp_ms += 1000 / kLowFps;
6977 }
6978
6979 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006981 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006982 video_source_.IncomingCapturedFrame(
6983 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6984 WaitForEncodedFrame(timestamp_ms);
6985 timestamp_ms += 1000 / kLowFps;
6986
6987 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6988
6989 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006990 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006991 const int kFrameIntervalMs = 1000 / kHighFps;
6992 max_framerate_ = kHighFps;
6993 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6994 video_source_.IncomingCapturedFrame(
6995 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6996 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6997 // be dropped if the encoder hans't been updated with the new higher target
6998 // framerate yet, causing it to overshoot the target bitrate and then
6999 // suffering the wrath of the media optimizer.
7000 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7001 timestamp_ms += kFrameIntervalMs;
7002 }
7003
7004 // Don expect correct measurement just yet, but it should be higher than
7005 // before.
7006 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7007
mflodmancc3d4422017-08-03 08:27:51 -07007008 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007009}
7010
mflodmancc3d4422017-08-03 08:27:51 -07007011TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007012 const int kFrameWidth = 1280;
7013 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007014 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007015 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007016 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007017
Henrik Boström381d1092020-05-12 18:49:07 +02007018 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007019 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007020 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007021
7022 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007023 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007024 video_source_.IncomingCapturedFrame(
7025 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7026 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007027 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007028
7029 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007031 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007032
7033 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007034 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007035 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007036
Per Kjellanderdcef6412020-10-07 15:09:05 +02007037 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007038 video_source_.IncomingCapturedFrame(
7039 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7040 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007041 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007042
mflodmancc3d4422017-08-03 08:27:51 -07007043 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007044}
ilnik6b826ef2017-06-16 06:53:48 -07007045
Niels Möller4db138e2018-04-19 09:04:13 +02007046TEST_F(VideoStreamEncoderTest,
7047 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7048 const int kFrameWidth = 1280;
7049 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007050 const test::ScopedKeyValueConfig kFieldTrials;
7051 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007052 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007053 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007054 video_source_.IncomingCapturedFrame(
7055 CreateFrame(1, kFrameWidth, kFrameHeight));
7056 WaitForEncodedFrame(1);
7057 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7058 .low_encode_usage_threshold_percent,
7059 default_options.low_encode_usage_threshold_percent);
7060 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7061 .high_encode_usage_threshold_percent,
7062 default_options.high_encode_usage_threshold_percent);
7063 video_stream_encoder_->Stop();
7064}
7065
7066TEST_F(VideoStreamEncoderTest,
7067 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7068 const int kFrameWidth = 1280;
7069 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007070 const test::ScopedKeyValueConfig kFieldTrials;
7071 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007072 hardware_options.low_encode_usage_threshold_percent = 150;
7073 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007074 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007075
Henrik Boström381d1092020-05-12 18:49:07 +02007076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007077 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007078 video_source_.IncomingCapturedFrame(
7079 CreateFrame(1, kFrameWidth, kFrameHeight));
7080 WaitForEncodedFrame(1);
7081 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7082 .low_encode_usage_threshold_percent,
7083 hardware_options.low_encode_usage_threshold_percent);
7084 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7085 .high_encode_usage_threshold_percent,
7086 hardware_options.high_encode_usage_threshold_percent);
7087 video_stream_encoder_->Stop();
7088}
7089
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007090TEST_F(VideoStreamEncoderTest,
7091 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7092 const int kFrameWidth = 1280;
7093 const int kFrameHeight = 720;
7094
Markus Handell8e4197b2022-05-30 15:45:28 +02007095 const test::ScopedKeyValueConfig kFieldTrials;
7096 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007097 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007098 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007099 video_source_.IncomingCapturedFrame(
7100 CreateFrame(1, kFrameWidth, kFrameHeight));
7101 WaitForEncodedFrame(1);
7102 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7103 .low_encode_usage_threshold_percent,
7104 default_options.low_encode_usage_threshold_percent);
7105 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7106 .high_encode_usage_threshold_percent,
7107 default_options.high_encode_usage_threshold_percent);
7108
Markus Handell8e4197b2022-05-30 15:45:28 +02007109 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007110 hardware_options.low_encode_usage_threshold_percent = 150;
7111 hardware_options.high_encode_usage_threshold_percent = 200;
7112 fake_encoder_.SetIsHardwareAccelerated(true);
7113
7114 video_source_.IncomingCapturedFrame(
7115 CreateFrame(2, kFrameWidth, kFrameHeight));
7116 WaitForEncodedFrame(2);
7117
7118 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7119 .low_encode_usage_threshold_percent,
7120 hardware_options.low_encode_usage_threshold_percent);
7121 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7122 .high_encode_usage_threshold_percent,
7123 hardware_options.high_encode_usage_threshold_percent);
7124
7125 video_stream_encoder_->Stop();
7126}
7127
Niels Möller6bb5ab92019-01-11 11:11:10 +01007128TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7129 const int kFrameWidth = 320;
7130 const int kFrameHeight = 240;
7131 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007132 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007133 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7134
Henrik Boström381d1092020-05-12 18:49:07 +02007135 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007136 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007137
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007138 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007139 max_framerate_ = kFps;
7140
7141 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7142 fake_encoder_.SimulateOvershoot(1.0);
7143 int num_dropped = 0;
7144 for (int i = 0; i < kNumFramesInRun; ++i) {
7145 video_source_.IncomingCapturedFrame(
7146 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7147 // Wait up to two frame durations for a frame to arrive.
7148 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7149 ++num_dropped;
7150 }
7151 timestamp_ms += 1000 / kFps;
7152 }
7153
Erik Språnga8d48ab2019-02-08 14:17:40 +01007154 // Framerate should be measured to be near the expected target rate.
7155 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7156
7157 // Frame drops should be within 5% of expected 0%.
7158 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007159
7160 // Make encoder produce frames at double the expected bitrate during 3 seconds
7161 // of video, verify number of drops. Rate needs to be slightly changed in
7162 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007163 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007164 const RateControlSettings trials =
7165 RateControlSettings::ParseFromFieldTrials();
7166 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007167 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007168 // frame dropping since the adjuter will try to just lower the target
7169 // bitrate rather than drop frames. If network headroom can be used, it
7170 // doesn't push back as hard so we don't need quite as much overshoot.
7171 // These numbers are unfortunately a bit magical but there's not trivial
7172 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007173 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007174 }
7175 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007177 kTargetBitrate + DataRate::KilobitsPerSec(1),
7178 kTargetBitrate + DataRate::KilobitsPerSec(1),
7179 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007180 num_dropped = 0;
7181 for (int i = 0; i < kNumFramesInRun; ++i) {
7182 video_source_.IncomingCapturedFrame(
7183 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7184 // Wait up to two frame durations for a frame to arrive.
7185 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7186 ++num_dropped;
7187 }
7188 timestamp_ms += 1000 / kFps;
7189 }
7190
Henrik Boström381d1092020-05-12 18:49:07 +02007191 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007192 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007193
7194 // Target framerate should be still be near the expected target, despite
7195 // the frame drops.
7196 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7197
7198 // Frame drops should be within 5% of expected 50%.
7199 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007200
7201 video_stream_encoder_->Stop();
7202}
7203
7204TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7205 const int kFrameWidth = 320;
7206 const int kFrameHeight = 240;
7207 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007208 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007209
7210 ASSERT_GT(max_framerate_, kActualInputFps);
7211
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007212 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007213 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007214 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007215 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007216
7217 // Insert 3 seconds of video, with an input fps lower than configured max.
7218 for (int i = 0; i < kActualInputFps * 3; ++i) {
7219 video_source_.IncomingCapturedFrame(
7220 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7221 // Wait up to two frame durations for a frame to arrive.
7222 WaitForEncodedFrame(timestamp_ms);
7223 timestamp_ms += 1000 / kActualInputFps;
7224 }
7225
7226 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7227
7228 video_stream_encoder_->Stop();
7229}
7230
Markus Handell9a478b52021-11-18 16:07:01 +01007231TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007232 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007233 test::FrameForwarder source;
7234 video_stream_encoder_->SetSource(&source,
7235 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007236 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007237 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007238
Markus Handell9a478b52021-11-18 16:07:01 +01007239 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007240 WaitForEncodedFrame(1);
7241 // On the very first frame full update should be forced.
7242 rect = fake_encoder_.GetLastUpdateRect();
7243 EXPECT_EQ(rect.offset_x, 0);
7244 EXPECT_EQ(rect.offset_y, 0);
7245 EXPECT_EQ(rect.height, codec_height_);
7246 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007247 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7248 // scheduled for processing during encoder queue processing of frame 2.
7249 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7250 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007251 WaitForEncodedFrame(3);
7252 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7253 rect = fake_encoder_.GetLastUpdateRect();
7254 EXPECT_EQ(rect.offset_x, 1);
7255 EXPECT_EQ(rect.offset_y, 0);
7256 EXPECT_EQ(rect.width, 10);
7257 EXPECT_EQ(rect.height, 1);
7258
Markus Handell9a478b52021-11-18 16:07:01 +01007259 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007260 WaitForEncodedFrame(4);
7261 // Previous frame was encoded, so no accumulation should happen.
7262 rect = fake_encoder_.GetLastUpdateRect();
7263 EXPECT_EQ(rect.offset_x, 0);
7264 EXPECT_EQ(rect.offset_y, 0);
7265 EXPECT_EQ(rect.width, 1);
7266 EXPECT_EQ(rect.height, 1);
7267
7268 video_stream_encoder_->Stop();
7269}
7270
Erik Språngd7329ca2019-02-21 21:19:53 +01007271TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007273 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007274
7275 // First frame is always keyframe.
7276 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7277 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007278 EXPECT_THAT(
7279 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007280 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007281
7282 // Insert delta frame.
7283 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7284 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007285 EXPECT_THAT(
7286 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007287 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007288
7289 // Request next frame be a key-frame.
7290 video_stream_encoder_->SendKeyFrame();
7291 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7292 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007293 EXPECT_THAT(
7294 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007295 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007296
7297 video_stream_encoder_->Stop();
7298}
7299
7300TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7301 // Setup simulcast with three streams.
7302 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007303 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007304 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7305 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007306 // Wait for all three layers before triggering event.
7307 sink_.SetNumExpectedLayers(3);
7308
7309 // First frame is always keyframe.
7310 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7311 WaitForEncodedFrame(1);
7312 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007313 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7314 VideoFrameType::kVideoFrameKey,
7315 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007316
7317 // Insert delta frame.
7318 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7319 WaitForEncodedFrame(2);
7320 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007321 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7322 VideoFrameType::kVideoFrameDelta,
7323 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007324
7325 // Request next frame be a key-frame.
7326 // Only first stream is configured to produce key-frame.
7327 video_stream_encoder_->SendKeyFrame();
7328 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7329 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007330
7331 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7332 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007333 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007334 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007335 VideoFrameType::kVideoFrameKey,
7336 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007337
7338 video_stream_encoder_->Stop();
7339}
7340
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007341TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007342 // SPS contains VUI with restrictions on the maximum number of reordered
7343 // pictures, there is no need to rewrite the bitstream to enable faster
7344 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345 ResetEncoder("H264", 1, 1, 1, false);
7346
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007350
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007351 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007352 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007353
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007354 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7355 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007356
7357 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007358 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007359
7360 video_stream_encoder_->Stop();
7361}
7362
7363TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007364 // SPS does not contain VUI, the bitstream is will be rewritten with added
7365 // VUI with restrictions on the maximum number of reordered pictures to
7366 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007367 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7368 0x00, 0x00, 0x03, 0x03, 0xF4,
7369 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007370 ResetEncoder("H264", 1, 1, 1, false);
7371
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007373 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007374 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007375
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007376 fake_encoder_.SetEncodedImageData(
7377 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007378
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007379 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7380 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007381
7382 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007383 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007384
7385 video_stream_encoder_->Stop();
7386}
7387
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007388TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7389 const int kFrameWidth = 1280;
7390 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007391 const DataRate kTargetBitrate =
7392 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007393
Henrik Boström381d1092020-05-12 18:49:07 +02007394 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007395 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007396 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7397
7398 // Insert a first video frame. It should be dropped because of downscale in
7399 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007400 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007401 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7402 frame.set_rotation(kVideoRotation_270);
7403 video_source_.IncomingCapturedFrame(frame);
7404
7405 ExpectDroppedFrame();
7406
7407 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007408 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007409 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7410 frame.set_rotation(kVideoRotation_90);
7411 video_source_.IncomingCapturedFrame(frame);
7412
7413 WaitForEncodedFrame(timestamp_ms);
7414 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7415
7416 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007417 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007418 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7419 frame.set_rotation(kVideoRotation_180);
7420 video_source_.IncomingCapturedFrame(frame);
7421
7422 WaitForEncodedFrame(timestamp_ms);
7423 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7424
7425 video_stream_encoder_->Stop();
7426}
7427
Erik Språng5056af02019-09-02 15:53:11 +02007428TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7429 const int kFrameWidth = 320;
7430 const int kFrameHeight = 180;
7431
7432 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007434 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7435 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7436 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007437 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007438 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007439 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007440
7441 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007442 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007443 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7444 frame.set_rotation(kVideoRotation_270);
7445 video_source_.IncomingCapturedFrame(frame);
7446 WaitForEncodedFrame(timestamp_ms);
7447
7448 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007449 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007450 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7451 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007453 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007454 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007455 /*link_allocation=*/target_rate,
7456 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007457 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007458 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007459 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7460
7461 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7462 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7463 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007464 DataRate allocation_sum =
7465 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007466 EXPECT_EQ(min_rate, allocation_sum);
7467 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7468
7469 video_stream_encoder_->Stop();
7470}
7471
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007472TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007474 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007475 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007476 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007477 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7478 WaitForEncodedFrame(1);
7479
7480 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7481 ASSERT_TRUE(prev_rate_settings.has_value());
7482 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7483 kDefaultFramerate);
7484
7485 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7486 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7487 timestamp_ms += 1000 / kDefaultFramerate;
7488 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7489 WaitForEncodedFrame(timestamp_ms);
7490 }
7491 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7492 kDefaultFramerate);
7493 // Capture larger frame to trigger a reconfigure.
7494 codec_height_ *= 2;
7495 codec_width_ *= 2;
7496 timestamp_ms += 1000 / kDefaultFramerate;
7497 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7498 WaitForEncodedFrame(timestamp_ms);
7499
7500 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7501 auto current_rate_settings =
7502 fake_encoder_.GetAndResetLastRateControlSettings();
7503 // Ensure we have actually reconfigured twice
7504 // The rate settings should have been set again even though
7505 // they haven't changed.
7506 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007507 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007508
7509 video_stream_encoder_->Stop();
7510}
7511
philipeld9cc8c02019-09-16 14:53:40 +02007512struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007513 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007514 MOCK_METHOD(void,
7515 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007516 (const webrtc::SdpVideoFormat& format,
7517 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007518 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007519};
7520
philipel9b058032020-02-10 11:30:00 +01007521TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7522 constexpr int kDontCare = 100;
7523 StrictMock<MockEncoderSelector> encoder_selector;
7524 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7525 &fake_encoder_, &encoder_selector);
7526 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7527
7528 // Reset encoder for new configuration to take effect.
7529 ConfigureEncoder(video_encoder_config_.Copy());
7530
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007531 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007532
7533 video_source_.IncomingCapturedFrame(
7534 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007535 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007536 video_stream_encoder_->Stop();
7537
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007538 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007539 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007540 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7541 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007542 video_stream_encoder_.reset();
7543}
7544
7545TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7546 constexpr int kDontCare = 100;
7547
7548 NiceMock<MockEncoderSelector> encoder_selector;
7549 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7550 video_send_config_.encoder_settings.encoder_switch_request_callback =
7551 &switch_callback;
7552 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7553 &fake_encoder_, &encoder_selector);
7554 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7555
7556 // Reset encoder for new configuration to take effect.
7557 ConfigureEncoder(video_encoder_config_.Copy());
7558
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007559 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007560 .WillByDefault(Return(SdpVideoFormat("AV1")));
7561 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007562 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7563 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007564
Henrik Boström381d1092020-05-12 18:49:07 +02007565 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007566 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7567 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7568 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007569 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007570 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007571 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007572 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007573
7574 video_stream_encoder_->Stop();
7575}
7576
philipel6daa3042022-04-11 10:48:28 +02007577TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7578 NiceMock<MockEncoderSelector> encoder_selector;
7579 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7580 video_send_config_.encoder_settings.encoder_switch_request_callback =
7581 &switch_callback;
7582 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7583 &fake_encoder_, &encoder_selector);
7584 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7585
7586 // Reset encoder for new configuration to take effect.
7587 ConfigureEncoder(video_encoder_config_.Copy());
7588
7589 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7590 .WillOnce(Return(absl::nullopt));
7591 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7592 .WillOnce(Return(SdpVideoFormat("AV1")));
7593 EXPECT_CALL(switch_callback,
7594 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7595 /*allow_default_fallback=*/false));
7596
7597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7598 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7599 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7600 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7601 /*fraction_lost=*/0,
7602 /*round_trip_time_ms=*/0,
7603 /*cwnd_reduce_ratio=*/0);
7604
7605 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7606 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7607 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7608
7609 AdvanceTime(TimeDelta::Zero());
7610
7611 video_stream_encoder_->Stop();
7612}
7613
philipel9b058032020-02-10 11:30:00 +01007614TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7615 constexpr int kSufficientBitrateToNotDrop = 1000;
7616 constexpr int kDontCare = 100;
7617
7618 NiceMock<MockVideoEncoder> video_encoder;
7619 NiceMock<MockEncoderSelector> encoder_selector;
7620 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7621 video_send_config_.encoder_settings.encoder_switch_request_callback =
7622 &switch_callback;
7623 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7624 &video_encoder, &encoder_selector);
7625 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7626
7627 // Reset encoder for new configuration to take effect.
7628 ConfigureEncoder(video_encoder_config_.Copy());
7629
7630 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7631 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7632 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007634 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7635 /*stable_target_bitrate=*/
7636 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7637 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007638 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007639 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007640 /*cwnd_reduce_ratio=*/0);
7641
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007642 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007643 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007644 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007645 .WillByDefault(Return(SdpVideoFormat("AV2")));
7646
7647 rtc::Event encode_attempted;
7648 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007649 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7650 /*allow_default_fallback=*/true))
7651 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007652
7653 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7654 encode_attempted.Wait(3000);
7655
Markus Handell28c71802021-11-08 10:11:55 +01007656 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007657
philipel9b058032020-02-10 11:30:00 +01007658 video_stream_encoder_->Stop();
7659
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007660 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7661 // to it's factory, so in order for the encoder instance in the
7662 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7663 // reset the `video_stream_encoder_` here.
7664 video_stream_encoder_.reset();
7665}
7666
7667TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7668 NiceMock<MockVideoEncoder> video_encoder;
7669 NiceMock<MockEncoderSelector> encoder_selector;
7670 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7671 video_send_config_.encoder_settings.encoder_switch_request_callback =
7672 &switch_callback;
7673 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7674 &video_encoder, &encoder_selector);
7675 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7676
7677 // Reset encoder for new configuration to take effect.
7678 ConfigureEncoder(video_encoder_config_.Copy());
7679
7680 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7681 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7682 /*round_trip_time_ms=*/0,
7683 /*cwnd_reduce_ratio=*/0);
7684 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7685
7686 ON_CALL(video_encoder, InitEncode(_, _))
7687 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7688 ON_CALL(encoder_selector, OnEncoderBroken)
7689 .WillByDefault(Return(SdpVideoFormat("AV2")));
7690
7691 rtc::Event encode_attempted;
7692 EXPECT_CALL(switch_callback,
7693 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7694 /*allow_default_fallback=*/true))
7695 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7696
7697 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7698 encode_attempted.Wait(3000);
7699
7700 AdvanceTime(TimeDelta::Zero());
7701
7702 video_stream_encoder_->Stop();
7703
7704 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7705 // to it's factory, so in order for the encoder instance in the
7706 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7707 // reset the `video_stream_encoder_` here.
7708 video_stream_encoder_.reset();
7709}
7710
7711TEST_F(VideoStreamEncoderTest,
7712 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7713 NiceMock<MockVideoEncoder> video_encoder;
7714 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7715 video_send_config_.encoder_settings.encoder_switch_request_callback =
7716 &switch_callback;
7717 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7718 &video_encoder, /*encoder_selector=*/nullptr);
7719 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7720
7721 // Reset encoder for new configuration to take effect.
7722 ConfigureEncoder(video_encoder_config_.Copy());
7723
7724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7725 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7726 /*round_trip_time_ms=*/0,
7727 /*cwnd_reduce_ratio=*/0);
7728 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7729
7730 ON_CALL(video_encoder, InitEncode(_, _))
7731 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7732
7733 rtc::Event encode_attempted;
7734 EXPECT_CALL(switch_callback,
7735 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7736 /*allow_default_fallback=*/true))
7737 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7738
7739 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7740 encode_attempted.Wait(3000);
7741
7742 AdvanceTime(TimeDelta::Zero());
7743
7744 video_stream_encoder_->Stop();
7745
7746 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007747 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007748 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7749 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007750 video_stream_encoder_.reset();
7751}
7752
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007753TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7754 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7755 // VideoEncoder is passed in encoder_factory, it checks whether
7756 // Codec Switch occurs without a crash.
7757 constexpr int kSufficientBitrateToNotDrop = 1000;
7758 constexpr int kDontCare = 100;
7759
7760 NiceMock<MockEncoderSelector> encoder_selector;
7761 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7762 video_send_config_.encoder_settings.encoder_switch_request_callback =
7763 &switch_callback;
7764 auto encoder_factory =
7765 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7766 /*encoder=*/nullptr, &encoder_selector);
7767 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7768
7769 // Reset encoder for new configuration to take effect.
7770 ConfigureEncoder(video_encoder_config_.Copy());
7771 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7772 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7773 // not fail.
7774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7775 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7776 /*stable_target_bitrate=*/
7777 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7778 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7779 /*fraction_lost=*/0,
7780 /*round_trip_time_ms=*/0,
7781 /*cwnd_reduce_ratio=*/0);
7782 ON_CALL(encoder_selector, OnEncoderBroken)
7783 .WillByDefault(Return(SdpVideoFormat("AV2")));
7784 rtc::Event encode_attempted;
7785 EXPECT_CALL(switch_callback,
7786 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7787 /*allow_default_fallback=*/_))
7788 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7789
7790 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7791 encode_attempted.Wait(3000);
7792
7793 AdvanceTime(TimeDelta::Zero());
7794
7795 video_stream_encoder_->Stop();
7796
7797 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7798 // to it's factory, so in order for the encoder instance in the
7799 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7800 // reset the `video_stream_encoder_` here.
7801 video_stream_encoder_.reset();
7802}
7803
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007804TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007805 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007806 const int kFrameWidth = 320;
7807 const int kFrameHeight = 180;
7808
7809 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007810 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007812 /*target_bitrate=*/rate,
7813 /*stable_target_bitrate=*/rate,
7814 /*link_allocation=*/rate,
7815 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007816 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007817 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007818
7819 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007820 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007821 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7822 frame.set_rotation(kVideoRotation_270);
7823 video_source_.IncomingCapturedFrame(frame);
7824 WaitForEncodedFrame(timestamp_ms);
7825 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7826
7827 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007828 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007830 /*target_bitrate=*/new_stable_rate,
7831 /*stable_target_bitrate=*/new_stable_rate,
7832 /*link_allocation=*/rate,
7833 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007834 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007835 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007836 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7837 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7838 video_stream_encoder_->Stop();
7839}
7840
7841TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007842 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007843 const int kFrameWidth = 320;
7844 const int kFrameHeight = 180;
7845
7846 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007847 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007849 /*target_bitrate=*/rate,
7850 /*stable_target_bitrate=*/rate,
7851 /*link_allocation=*/rate,
7852 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007853 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007854 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007855
7856 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007857 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007858 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7859 frame.set_rotation(kVideoRotation_270);
7860 video_source_.IncomingCapturedFrame(frame);
7861 WaitForEncodedFrame(timestamp_ms);
7862 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7863
7864 // Set a higher target rate without changing the link_allocation. Should not
7865 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007866 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007867 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007868 /*target_bitrate=*/rate,
7869 /*stable_target_bitrate=*/new_stable_rate,
7870 /*link_allocation=*/rate,
7871 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007872 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007873 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007874 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7875 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7876 video_stream_encoder_->Stop();
7877}
7878
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007879TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007880 test::ScopedKeyValueConfig field_trials(
7881 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007882 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7883 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7884 const int kFramerateFps = 30;
7885 const int kWidth = 1920;
7886 const int kHeight = 1080;
7887 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7888 // Works on screenshare mode.
7889 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7890 // We rely on the automatic resolution adaptation, but we handle framerate
7891 // adaptation manually by mocking the stats proxy.
7892 video_source_.set_adaptation_enabled(true);
7893
7894 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007897 video_stream_encoder_->SetSource(&video_source_,
7898 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007899 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007900
7901 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7902 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7903
7904 // Pass enough frames with the full update to trigger animation detection.
7905 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007906 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007907 frame.set_ntp_time_ms(timestamp_ms);
7908 frame.set_timestamp_us(timestamp_ms * 1000);
7909 video_source_.IncomingCapturedFrame(frame);
7910 WaitForEncodedFrame(timestamp_ms);
7911 }
7912
7913 // Resolution should be limited.
7914 rtc::VideoSinkWants expected;
7915 expected.max_framerate_fps = kFramerateFps;
7916 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007917 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007918
7919 // Pass one frame with no known update.
7920 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007921 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007922 frame.set_ntp_time_ms(timestamp_ms);
7923 frame.set_timestamp_us(timestamp_ms * 1000);
7924 frame.clear_update_rect();
7925
7926 video_source_.IncomingCapturedFrame(frame);
7927 WaitForEncodedFrame(timestamp_ms);
7928
7929 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007930 EXPECT_THAT(video_source_.sink_wants(),
7931 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007932
7933 video_stream_encoder_->Stop();
7934}
7935
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007936TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7937 const int kWidth = 720; // 540p adapted down.
7938 const int kHeight = 405;
7939 const int kNumFrames = 3;
7940 // Works on screenshare mode.
7941 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7942 /*num_spatial_layers=*/2, /*screenshare=*/true);
7943
7944 video_source_.set_adaptation_enabled(true);
7945
7946 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007947 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007948
7949 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7950
7951 // Pass enough frames with the full update to trigger animation detection.
7952 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007953 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007954 frame.set_ntp_time_ms(timestamp_ms);
7955 frame.set_timestamp_us(timestamp_ms * 1000);
7956 video_source_.IncomingCapturedFrame(frame);
7957 WaitForEncodedFrame(timestamp_ms);
7958 }
7959
7960 video_stream_encoder_->Stop();
7961}
7962
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007963TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7964 const float downscale_factors[] = {4.0, 2.0, 1.0};
7965 const int number_layers =
7966 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7967 VideoEncoderConfig config;
7968 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7969 for (int i = 0; i < number_layers; ++i) {
7970 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7971 config.simulcast_layers[i].active = true;
7972 }
7973 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007974 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007975 "VP8", /*max qp*/ 56, /*screencast*/ false,
7976 /*screenshare enabled*/ false);
7977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007978 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7979 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007980
7981 // First initialization.
7982 // Encoder should be initialized. Next frame should be key frame.
7983 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7984 sink_.SetNumExpectedLayers(number_layers);
7985 int64_t timestamp_ms = kFrameIntervalMs;
7986 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7987 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007988 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007989 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7990 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7991 VideoFrameType::kVideoFrameKey,
7992 VideoFrameType::kVideoFrameKey}));
7993
7994 // Disable top layer.
7995 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7996 config.simulcast_layers[number_layers - 1].active = false;
7997 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7998 sink_.SetNumExpectedLayers(number_layers - 1);
7999 timestamp_ms += kFrameIntervalMs;
8000 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8001 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008002 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008003 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8004 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8005 VideoFrameType::kVideoFrameDelta,
8006 VideoFrameType::kVideoFrameDelta}));
8007
8008 // Re-enable top layer.
8009 // Encoder should be re-initialized. Next frame should be key frame.
8010 config.simulcast_layers[number_layers - 1].active = true;
8011 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8012 sink_.SetNumExpectedLayers(number_layers);
8013 timestamp_ms += kFrameIntervalMs;
8014 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8015 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008016 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008017 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8018 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8019 VideoFrameType::kVideoFrameKey,
8020 VideoFrameType::kVideoFrameKey}));
8021
8022 // Top layer max rate change.
8023 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8024 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8025 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8026 sink_.SetNumExpectedLayers(number_layers);
8027 timestamp_ms += kFrameIntervalMs;
8028 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8029 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008030 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008031 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8032 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8033 VideoFrameType::kVideoFrameDelta,
8034 VideoFrameType::kVideoFrameDelta}));
8035
8036 // Top layer resolution change.
8037 // Encoder should be re-initialized. Next frame should be key frame.
8038 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8039 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8040 sink_.SetNumExpectedLayers(number_layers);
8041 timestamp_ms += kFrameIntervalMs;
8042 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8043 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008044 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008045 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8046 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8047 VideoFrameType::kVideoFrameKey,
8048 VideoFrameType::kVideoFrameKey}));
8049 video_stream_encoder_->Stop();
8050}
8051
Henrik Boström1124ed12021-02-25 10:30:39 +01008052TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8053 const int kFrameWidth = 1280;
8054 const int kFrameHeight = 720;
8055
8056 SetUp();
8057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008058 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008059
8060 // Capturing a frame should reconfigure the encoder and expose the encoder
8061 // resolution, which is the same as the input frame.
8062 int64_t timestamp_ms = kFrameIntervalMs;
8063 video_source_.IncomingCapturedFrame(
8064 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8065 WaitForEncodedFrame(timestamp_ms);
8066 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8067 EXPECT_THAT(video_source_.sink_wants().resolutions,
8068 ::testing::ElementsAreArray(
8069 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8070
8071 video_stream_encoder_->Stop();
8072}
8073
8074TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8075 // Pick downscale factors such that we never encode at full resolution - this
8076 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008077 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008078 // encoder should not ask for the frame resolution. This allows video frames
8079 // to have the appearence of one resolution but optimize its internal buffers
8080 // for what is actually encoded.
8081 const size_t kNumSimulcastLayers = 3u;
8082 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8083 const int kFrameWidth = 1280;
8084 const int kFrameHeight = 720;
8085 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8086 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8087 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8088 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8089 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8090 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8091
8092 VideoEncoderConfig config;
8093 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8094 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8095 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8096 config.simulcast_layers[i].active = true;
8097 }
8098 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008099 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008100 "VP8", /*max qp*/ 56, /*screencast*/ false,
8101 /*screenshare enabled*/ false);
8102 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008103 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8104 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008105
8106 // Capture a frame with all layers active.
8107 int64_t timestamp_ms = kFrameIntervalMs;
8108 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8109 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8110 video_source_.IncomingCapturedFrame(
8111 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8112 WaitForEncodedFrame(timestamp_ms);
8113 // Expect encoded resolutions to match the expected simulcast layers.
8114 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8115 EXPECT_THAT(
8116 video_source_.sink_wants().resolutions,
8117 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8118
8119 // Capture a frame with one of the layers inactive.
8120 timestamp_ms += kFrameIntervalMs;
8121 config.simulcast_layers[2].active = false;
8122 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8123 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8124 video_source_.IncomingCapturedFrame(
8125 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8126 WaitForEncodedFrame(timestamp_ms);
8127
8128 // Expect encoded resolutions to match the expected simulcast layers.
8129 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8130 EXPECT_THAT(video_source_.sink_wants().resolutions,
8131 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8132
8133 // Capture a frame with all but one layer turned off.
8134 timestamp_ms += kFrameIntervalMs;
8135 config.simulcast_layers[1].active = false;
8136 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8137 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8138 video_source_.IncomingCapturedFrame(
8139 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8140 WaitForEncodedFrame(timestamp_ms);
8141
8142 // Expect encoded resolutions to match the expected simulcast layers.
8143 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8144 EXPECT_THAT(video_source_.sink_wants().resolutions,
8145 ::testing::ElementsAreArray({kLayer0Size}));
8146
8147 video_stream_encoder_->Stop();
8148}
8149
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008150TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008151 ResetEncoder("VP8", 1, 1, 1, false);
8152
Niels Möller8b692902021-06-14 12:04:57 +02008153 // Force encoder reconfig.
8154 video_source_.IncomingCapturedFrame(
8155 CreateFrame(1, codec_width_, codec_height_));
8156 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8157
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008158 // Set QP on encoded frame and pass the frame to encode complete callback.
8159 // Since QP is present QP parsing won't be triggered and the original value
8160 // should be kept.
8161 EncodedImage encoded_image;
8162 encoded_image.qp_ = 123;
8163 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8164 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8165 CodecSpecificInfo codec_info;
8166 codec_info.codecType = kVideoCodecVP8;
8167 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8168 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8169 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8170 video_stream_encoder_->Stop();
8171}
8172
8173TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008174 ResetEncoder("VP8", 1, 1, 1, false);
8175
Niels Möller8b692902021-06-14 12:04:57 +02008176 // Force encoder reconfig.
8177 video_source_.IncomingCapturedFrame(
8178 CreateFrame(1, codec_width_, codec_height_));
8179 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8180
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008181 // Pass an encoded frame without QP to encode complete callback. QP should be
8182 // parsed and set.
8183 EncodedImage encoded_image;
8184 encoded_image.qp_ = -1;
8185 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8186 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8187 CodecSpecificInfo codec_info;
8188 codec_info.codecType = kVideoCodecVP8;
8189 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8190 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8191 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8192 video_stream_encoder_->Stop();
8193}
8194
8195TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008196 webrtc::test::ScopedKeyValueConfig field_trials(
8197 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008198
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008199 ResetEncoder("VP8", 1, 1, 1, false);
8200
Niels Möller8b692902021-06-14 12:04:57 +02008201 // Force encoder reconfig.
8202 video_source_.IncomingCapturedFrame(
8203 CreateFrame(1, codec_width_, codec_height_));
8204 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8205
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008206 EncodedImage encoded_image;
8207 encoded_image.qp_ = -1;
8208 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8209 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8210 CodecSpecificInfo codec_info;
8211 codec_info.codecType = kVideoCodecVP8;
8212 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8213 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8214 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8215 video_stream_encoder_->Stop();
8216}
8217
Sergey Silkind19e3b92021-03-16 10:05:30 +00008218TEST_F(VideoStreamEncoderTest,
8219 QualityScalingNotAllowed_QualityScalingDisabled) {
8220 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8221
8222 // Disable scaling settings in encoder info.
8223 fake_encoder_.SetQualityScaling(false);
8224 // Disable quality scaling in encoder config.
8225 video_encoder_config.is_quality_scaling_allowed = false;
8226 ConfigureEncoder(std::move(video_encoder_config));
8227
8228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008229 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008230
8231 test::FrameForwarder source;
8232 video_stream_encoder_->SetSource(
8233 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8234 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8235 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8236
8237 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8238 WaitForEncodedFrame(1);
8239 video_stream_encoder_->TriggerQualityLow();
8240 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8241
8242 video_stream_encoder_->Stop();
8243}
8244
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008245TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8246 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8247
8248 // Disable scaling settings in encoder info.
8249 fake_encoder_.SetQualityScaling(false);
8250 // Set QP trusted in encoder info.
8251 fake_encoder_.SetIsQpTrusted(true);
8252 // Enable quality scaling in encoder config.
8253 video_encoder_config.is_quality_scaling_allowed = false;
8254 ConfigureEncoder(std::move(video_encoder_config));
8255
8256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008257 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008258
8259 test::FrameForwarder source;
8260 video_stream_encoder_->SetSource(
8261 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8262 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8263 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8264
8265 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8266 WaitForEncodedFrame(1);
8267 video_stream_encoder_->TriggerQualityLow();
8268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8269
8270 video_stream_encoder_->Stop();
8271}
8272
Shuhai Pengf2707702021-09-29 17:19:44 +08008273TEST_F(VideoStreamEncoderTest,
8274 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8275 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8276
8277 // Disable scaling settings in encoder info.
8278 fake_encoder_.SetQualityScaling(false);
8279 // Set QP trusted in encoder info.
8280 fake_encoder_.SetIsQpTrusted(true);
8281 // Enable quality scaling in encoder config.
8282 video_encoder_config.is_quality_scaling_allowed = false;
8283 ConfigureEncoder(std::move(video_encoder_config));
8284
8285 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008286 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008287
8288 test::FrameForwarder source;
8289 video_stream_encoder_->SetSource(
8290 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8291 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8293
8294 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8295 WaitForEncodedFrame(1);
8296 video_stream_encoder_->TriggerQualityLow();
8297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8298
8299 video_stream_encoder_->Stop();
8300}
8301
8302TEST_F(VideoStreamEncoderTest,
8303 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8304 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8305
8306 // Disable scaling settings in encoder info.
8307 fake_encoder_.SetQualityScaling(false);
8308 // Set QP trusted in encoder info.
8309 fake_encoder_.SetIsQpTrusted(false);
8310 // Enable quality scaling in encoder config.
8311 video_encoder_config.is_quality_scaling_allowed = false;
8312 ConfigureEncoder(std::move(video_encoder_config));
8313
8314 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008315 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008316
8317 test::FrameForwarder source;
8318 video_stream_encoder_->SetSource(
8319 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8320 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8321 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8322
8323 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8324 WaitForEncodedFrame(1);
8325 video_stream_encoder_->TriggerQualityLow();
8326 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8327
8328 video_stream_encoder_->Stop();
8329}
8330
8331TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8332 // Set QP trusted in encoder info.
8333 fake_encoder_.SetIsQpTrusted(false);
8334
8335 const int MinEncBitrateKbps = 30;
8336 const int MaxEncBitrateKbps = 100;
8337 const int MinStartBitrateKbp = 50;
8338 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8339 /*frame_size_pixels=*/codec_width_ * codec_height_,
8340 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8341 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8342 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8343
8344 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008345 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008346
8347 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8348
8349 VideoEncoderConfig video_encoder_config;
8350 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8351 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8352 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8353 MinEncBitrateKbps * 1000;
8354 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8355 kMaxPayloadLength);
8356
8357 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8358 WaitForEncodedFrame(1);
8359 EXPECT_EQ(
8360 MaxEncBitrateKbps,
8361 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8362 EXPECT_EQ(
8363 MinEncBitrateKbps,
8364 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8365
8366 video_stream_encoder_->Stop();
8367}
8368
8369TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8370 // Set QP trusted in encoder info.
8371 fake_encoder_.SetIsQpTrusted(false);
8372
8373 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8374 EncoderInfoSettings::
8375 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8376 codec_width_ * codec_height_,
8377 EncoderInfoSettings::
8378 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8379 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8380
8381 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8382 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8383 const int TargetEncBitrate = MaxEncBitrate;
8384 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8385 DataRate::BitsPerSec(TargetEncBitrate),
8386 DataRate::BitsPerSec(TargetEncBitrate),
8387 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8388
8389 VideoEncoderConfig video_encoder_config;
8390 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8391 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8392 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8393 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8394 kMaxPayloadLength);
8395
8396 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8397 WaitForEncodedFrame(1);
8398 EXPECT_EQ(
8399 MaxEncBitrate / 1000,
8400 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8401 EXPECT_EQ(
8402 MinEncBitrate / 1000,
8403 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8404
8405 video_stream_encoder_->Stop();
8406}
8407
Erik Språnge4589cb2022-04-06 16:44:30 +02008408TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8409 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8410 /*num_spatial_layers=*/1,
8411 /*screenshare=*/false, /*allocation_callback_type=*/
8412 VideoStreamEncoder::BitrateAllocationCallbackType::
8413 kVideoBitrateAllocationWhenScreenSharing,
8414 /*num_cores=*/3);
8415
8416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8417 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8418 video_source_.IncomingCapturedFrame(
8419 CreateFrame(1, /*width=*/320, /*height=*/180));
8420 WaitForEncodedFrame(1);
8421 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8422 VideoCodecComplexity::kComplexityNormal);
8423 video_stream_encoder_->Stop();
8424}
8425
8426TEST_F(VideoStreamEncoderTest,
8427 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8428 webrtc::test::ScopedKeyValueConfig field_trials(
8429 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8430
8431 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8432 /*num_spatial_layers=*/1,
8433 /*screenshare=*/false, /*allocation_callback_type=*/
8434 VideoStreamEncoder::BitrateAllocationCallbackType::
8435 kVideoBitrateAllocationWhenScreenSharing,
8436 /*num_cores=*/2);
8437
8438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8439 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8440 video_source_.IncomingCapturedFrame(
8441 CreateFrame(1, /*width=*/320, /*height=*/180));
8442 WaitForEncodedFrame(1);
8443 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8444 VideoCodecComplexity::kComplexityNormal);
8445 video_stream_encoder_->Stop();
8446}
8447
8448TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8449 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8450 /*num_spatial_layers=*/1,
8451 /*screenshare=*/false, /*allocation_callback_type=*/
8452 VideoStreamEncoder::BitrateAllocationCallbackType::
8453 kVideoBitrateAllocationWhenScreenSharing,
8454 /*num_cores=*/2);
8455
8456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8457 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8458 video_source_.IncomingCapturedFrame(
8459 CreateFrame(1, /*width=*/320, /*height=*/180));
8460 WaitForEncodedFrame(1);
8461 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8462 VideoCodecComplexity::kComplexityLow);
8463 video_stream_encoder_->Stop();
8464}
8465
Sergey Silkind19e3b92021-03-16 10:05:30 +00008466#if !defined(WEBRTC_IOS)
8467// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8468// disabled by default on iOS.
8469TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8470 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8471
8472 // Disable scaling settings in encoder info.
8473 fake_encoder_.SetQualityScaling(false);
8474 // Enable quality scaling in encoder config.
8475 video_encoder_config.is_quality_scaling_allowed = true;
8476 ConfigureEncoder(std::move(video_encoder_config));
8477
8478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008479 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008480
8481 test::FrameForwarder source;
8482 video_stream_encoder_->SetSource(
8483 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8484 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8485 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8486
8487 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8488 WaitForEncodedFrame(1);
8489 video_stream_encoder_->TriggerQualityLow();
8490 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8491
8492 video_stream_encoder_->Stop();
8493}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008494
8495TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8496 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8497
8498 // Disable scaling settings in encoder info.
8499 fake_encoder_.SetQualityScaling(false);
8500 // Set QP trusted in encoder info.
8501 fake_encoder_.SetIsQpTrusted(true);
8502 // Enable quality scaling in encoder config.
8503 video_encoder_config.is_quality_scaling_allowed = true;
8504 ConfigureEncoder(std::move(video_encoder_config));
8505
8506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008507 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008508
8509 test::FrameForwarder source;
8510 video_stream_encoder_->SetSource(
8511 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8512 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8514
8515 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8516 WaitForEncodedFrame(1);
8517 video_stream_encoder_->TriggerQualityLow();
8518 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8519
8520 video_stream_encoder_->Stop();
8521}
Shuhai Pengf2707702021-09-29 17:19:44 +08008522
8523TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8524 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8525
8526 // Disable scaling settings in encoder info.
8527 fake_encoder_.SetQualityScaling(false);
8528 // Set QP not trusted in encoder info.
8529 fake_encoder_.SetIsQpTrusted(false);
8530 // Enable quality scaling in encoder config.
8531 video_encoder_config.is_quality_scaling_allowed = true;
8532 ConfigureEncoder(std::move(video_encoder_config));
8533
8534 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008535 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008536
8537 test::FrameForwarder source;
8538 video_stream_encoder_->SetSource(
8539 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8540 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8542
8543 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8544 WaitForEncodedFrame(1);
8545 video_stream_encoder_->TriggerQualityLow();
8546 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8547 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8548 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8549
8550 video_stream_encoder_->Stop();
8551}
8552
8553TEST_F(VideoStreamEncoderTest,
8554 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8555 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8556
8557 // Disable scaling settings in encoder info.
8558 fake_encoder_.SetQualityScaling(false);
8559 // Set QP trusted in encoder info.
8560 fake_encoder_.SetIsQpTrusted(true);
8561 // Enable quality scaling in encoder config.
8562 video_encoder_config.is_quality_scaling_allowed = true;
8563 ConfigureEncoder(std::move(video_encoder_config));
8564
8565 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008566 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008567
8568 test::FrameForwarder source;
8569 video_stream_encoder_->SetSource(
8570 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8571 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8572 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8573
8574 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8575 WaitForEncodedFrame(1);
8576 video_stream_encoder_->TriggerQualityLow();
8577 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8579
8580 video_stream_encoder_->Stop();
8581}
8582
8583TEST_F(VideoStreamEncoderTest,
8584 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8585 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8586
8587 // Disable scaling settings in encoder info.
8588 fake_encoder_.SetQualityScaling(false);
8589 // Set QP trusted in encoder info.
8590 fake_encoder_.SetIsQpTrusted(false);
8591 // Enable quality scaling in encoder config.
8592 video_encoder_config.is_quality_scaling_allowed = true;
8593 ConfigureEncoder(std::move(video_encoder_config));
8594
8595 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008596 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008597
8598 test::FrameForwarder source;
8599 video_stream_encoder_->SetSource(
8600 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8601 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8602 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8603
8604 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8605 WaitForEncodedFrame(1);
8606 video_stream_encoder_->TriggerQualityLow();
8607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8608
8609 video_stream_encoder_->Stop();
8610}
8611
Erik Språng5e13d052022-08-02 11:42:49 +02008612TEST_F(VideoStreamEncoderTest,
8613 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8614 // Send a native frame before encoder rates have been set. The encoder is
8615 // seen as paused at this time.
8616 rtc::Event frame_destroyed_event;
8617 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8618 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8619
8620 // Frame should be dropped and destroyed.
8621 ExpectDroppedFrame();
8622 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
8623 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8624
8625 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8626 // frame.
8627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8628 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8629 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8630 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8631
8632 video_stream_encoder_->Stop();
8633}
8634
Erik Språnge4589cb2022-04-06 16:44:30 +02008635#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008636
Henrik Boström56db9ff2021-03-24 09:06:45 +01008637// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8638class VideoStreamEncoderWithRealEncoderTest
8639 : public VideoStreamEncoderTest,
8640 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8641 public:
8642 VideoStreamEncoderWithRealEncoderTest()
8643 : VideoStreamEncoderTest(),
8644 codec_type_(std::get<0>(GetParam())),
8645 allow_i420_conversion_(std::get<1>(GetParam())) {}
8646
8647 void SetUp() override {
8648 VideoStreamEncoderTest::SetUp();
8649 std::unique_ptr<VideoEncoder> encoder;
8650 switch (codec_type_) {
8651 case kVideoCodecVP8:
8652 encoder = VP8Encoder::Create();
8653 break;
8654 case kVideoCodecVP9:
8655 encoder = VP9Encoder::Create();
8656 break;
8657 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008658 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008659 break;
8660 case kVideoCodecH264:
8661 encoder =
8662 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8663 break;
8664 case kVideoCodecMultiplex:
8665 mock_encoder_factory_for_multiplex_ =
8666 std::make_unique<MockVideoEncoderFactory>();
8667 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8668 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8669 .WillRepeatedly([] { return VP8Encoder::Create(); });
8670 encoder = std::make_unique<MultiplexEncoderAdapter>(
8671 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8672 false);
8673 break;
8674 default:
Artem Titovd3251962021-11-15 16:57:07 +01008675 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008676 }
8677 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8678 }
8679
8680 void TearDown() override {
8681 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008682 // Ensure `video_stream_encoder_` is destroyed before
8683 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008684 video_stream_encoder_.reset();
8685 VideoStreamEncoderTest::TearDown();
8686 }
8687
8688 protected:
8689 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8690 std::unique_ptr<VideoEncoder> encoder) {
8691 // Configure VSE to use the encoder.
8692 encoder_ = std::move(encoder);
8693 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8694 encoder_.get(), &encoder_selector_);
8695 video_send_config_.encoder_settings.encoder_factory =
8696 encoder_proxy_factory_.get();
8697 VideoEncoderConfig video_encoder_config;
8698 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8699 video_encoder_config_ = video_encoder_config.Copy();
8700 ConfigureEncoder(video_encoder_config_.Copy());
8701
8702 // Set bitrate to ensure frame is not dropped.
8703 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008704 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008705 }
8706
8707 const VideoCodecType codec_type_;
8708 const bool allow_i420_conversion_;
8709 NiceMock<MockEncoderSelector> encoder_selector_;
8710 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8711 std::unique_ptr<VideoEncoder> encoder_;
8712 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8713};
8714
8715TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8716 auto native_i420_frame = test::CreateMappableNativeFrame(
8717 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8718 video_source_.IncomingCapturedFrame(native_i420_frame);
8719 WaitForEncodedFrame(codec_width_, codec_height_);
8720
8721 auto mappable_native_buffer =
8722 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8723 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8724 mappable_native_buffer->GetMappedFramedBuffers();
8725 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8726 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8727 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8728 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8729}
8730
8731TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8732 auto native_nv12_frame = test::CreateMappableNativeFrame(
8733 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8734 video_source_.IncomingCapturedFrame(native_nv12_frame);
8735 WaitForEncodedFrame(codec_width_, codec_height_);
8736
8737 auto mappable_native_buffer =
8738 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8739 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8740 mappable_native_buffer->GetMappedFramedBuffers();
8741 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8742 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8743 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8744 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8745
8746 if (!allow_i420_conversion_) {
8747 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8748 }
8749}
8750
Erik Språng7444b192021-06-02 14:02:13 +02008751TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8752 if (codec_type_ == kVideoCodecMultiplex) {
8753 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8754 return;
8755 }
8756
8757 const size_t kNumSpatialLayers = 3u;
8758 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8759 const int kFrameWidth = 1280;
8760 const int kFrameHeight = 720;
8761 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8762 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8763 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8764 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8765 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8766 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8767
8768 VideoEncoderConfig config;
8769 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8770 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008771 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008772 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8773 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8774 vp9_settings.numberOfTemporalLayers = 3;
8775 vp9_settings.automaticResizeOn = false;
8776 config.encoder_specific_settings =
8777 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8778 vp9_settings);
8779 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8780 /*fps=*/30.0,
8781 /*first_active_layer=*/0,
8782 /*num_spatial_layers=*/3,
8783 /*num_temporal_layers=*/3,
8784 /*is_screenshare=*/false);
8785 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8786 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008787 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008788 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8789 /*fps=*/30.0,
8790 /*first_active_layer=*/0,
8791 /*num_spatial_layers=*/3,
8792 /*num_temporal_layers=*/3,
8793 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008794 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008795 } else {
8796 // Simulcast for VP8/H264.
8797 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8798 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8799 config.simulcast_layers[i].scale_resolution_down_by =
8800 kDownscaleFactors[i];
8801 config.simulcast_layers[i].active = true;
8802 }
8803 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8804 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008805 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008806 }
8807 }
8808
8809 auto set_layer_active = [&](int layer_idx, bool active) {
8810 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8811 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8812 config.spatial_layers[layer_idx].active = active;
8813 } else {
8814 config.simulcast_layers[layer_idx].active = active;
8815 }
8816 };
8817
8818 config.video_stream_factory =
8819 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8820 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8821 /*screencast*/ false,
8822 /*screenshare enabled*/ false);
8823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008824 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8825 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008826
8827 // Capture a frame with all layers active.
8828 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8829 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8830 int64_t timestamp_ms = kFrameIntervalMs;
8831 video_source_.IncomingCapturedFrame(
8832 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8833
8834 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8836
8837 // Capture a frame with one of the layers inactive.
8838 set_layer_active(2, false);
8839 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8840 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8841 timestamp_ms += kFrameIntervalMs;
8842 video_source_.IncomingCapturedFrame(
8843 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8844 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8845
8846 // New target bitrates signaled based on lower resolution.
8847 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8849 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8851
8852 // Re-enable the top layer.
8853 set_layer_active(2, true);
8854 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8855 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8856 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8857
8858 // Bitrate target adjusted back up to enable HD layer...
8859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8860 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8861 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8863
8864 // ...then add a new frame.
8865 timestamp_ms += kFrameIntervalMs;
8866 video_source_.IncomingCapturedFrame(
8867 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8868 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8869 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8870
8871 video_stream_encoder_->Stop();
8872}
8873
Henrik Boström56db9ff2021-03-24 09:06:45 +01008874std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8875 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8876 VideoCodecType codec_type = std::get<0>(info.param);
8877 bool allow_i420_conversion = std::get<1>(info.param);
8878 std::string str;
8879 switch (codec_type) {
8880 case kVideoCodecGeneric:
8881 str = "Generic";
8882 break;
8883 case kVideoCodecVP8:
8884 str = "VP8";
8885 break;
8886 case kVideoCodecVP9:
8887 str = "VP9";
8888 break;
8889 case kVideoCodecAV1:
8890 str = "AV1";
8891 break;
8892 case kVideoCodecH264:
8893 str = "H264";
8894 break;
8895 case kVideoCodecMultiplex:
8896 str = "Multiplex";
8897 break;
8898 default:
Artem Titovd3251962021-11-15 16:57:07 +01008899 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008900 }
8901 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8902 return str;
8903}
8904
8905constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8906 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8907constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8908 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8909constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008910 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008911constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8912 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8913#if defined(WEBRTC_USE_H264)
8914constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8915 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8916
8917// The windows compiler does not tolerate #if statements inside the
8918// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8919// and without H264).
8920INSTANTIATE_TEST_SUITE_P(
8921 All,
8922 VideoStreamEncoderWithRealEncoderTest,
8923 ::testing::Values(kVP8DisallowConversion,
8924 kVP9DisallowConversion,
8925 kAV1AllowConversion,
8926 kMultiplexDisallowConversion,
8927 kH264AllowConversion),
8928 TestParametersVideoCodecAndAllowI420ConversionToString);
8929#else
8930INSTANTIATE_TEST_SUITE_P(
8931 All,
8932 VideoStreamEncoderWithRealEncoderTest,
8933 ::testing::Values(kVP8DisallowConversion,
8934 kVP9DisallowConversion,
8935 kAV1AllowConversion,
8936 kMultiplexDisallowConversion),
8937 TestParametersVideoCodecAndAllowI420ConversionToString);
8938#endif
8939
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008940class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8941 protected:
8942 void RunTest(const std::vector<VideoStream>& configs,
8943 const int expected_num_init_encode) {
8944 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008945 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008946 InsertFrameAndWaitForEncoded();
8947 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8948 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008949 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8950 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008951
8952 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8953 ConfigureEncoder(configs[1]);
8954 InsertFrameAndWaitForEncoded();
8955 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8956 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008957 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008958 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008959 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008960
8961 video_stream_encoder_->Stop();
8962 }
8963
8964 void ConfigureEncoder(const VideoStream& stream) {
8965 VideoEncoderConfig config;
8966 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8967 config.max_bitrate_bps = stream.max_bitrate_bps;
8968 config.simulcast_layers[0] = stream;
8969 config.video_stream_factory =
8970 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8971 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8972 /*conference_mode=*/false);
8973 video_stream_encoder_->ConfigureEncoder(std::move(config),
8974 kMaxPayloadLength);
8975 }
8976
8977 void OnBitrateUpdated(DataRate bitrate) {
8978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8979 bitrate, bitrate, bitrate, 0, 0, 0);
8980 }
8981
8982 void InsertFrameAndWaitForEncoded() {
8983 timestamp_ms_ += kFrameIntervalMs;
8984 video_source_.IncomingCapturedFrame(
8985 CreateFrame(timestamp_ms_, kWidth, kHeight));
8986 sink_.WaitForEncodedFrame(timestamp_ms_);
8987 }
8988
8989 void ExpectEqual(const VideoCodec& actual,
8990 const VideoStream& expected) const {
8991 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8992 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8993 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8994 static_cast<unsigned int>(expected.min_bitrate_bps));
8995 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8996 static_cast<unsigned int>(expected.max_bitrate_bps));
8997 EXPECT_EQ(actual.simulcastStream[0].width,
8998 kWidth / expected.scale_resolution_down_by);
8999 EXPECT_EQ(actual.simulcastStream[0].height,
9000 kHeight / expected.scale_resolution_down_by);
9001 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9002 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009003 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009004 }
9005
9006 VideoStream DefaultConfig() const {
9007 VideoStream stream;
9008 stream.max_framerate = 25;
9009 stream.min_bitrate_bps = 35000;
9010 stream.max_bitrate_bps = 900000;
9011 stream.scale_resolution_down_by = 1.0;
9012 stream.num_temporal_layers = 1;
9013 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009014 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009015 return stream;
9016 }
9017
9018 const int kWidth = 640;
9019 const int kHeight = 360;
9020 int64_t timestamp_ms_ = 0;
9021};
9022
9023TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9024 VideoStream config1 = DefaultConfig();
9025 VideoStream config2 = config1;
9026 config2.max_framerate++;
9027
9028 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9029}
9030
9031TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9032 VideoStream config1 = DefaultConfig();
9033 VideoStream config2 = config1;
9034 config2.min_bitrate_bps += 10000;
9035
9036 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9037}
9038
9039TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9040 VideoStream config1 = DefaultConfig();
9041 VideoStream config2 = config1;
9042 config2.max_bitrate_bps += 100000;
9043
9044 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9045}
9046
9047TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9048 VideoStream config1 = DefaultConfig();
9049 VideoStream config2 = config1;
9050 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9051
9052 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9053}
9054
9055TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9056 VideoStream config1 = DefaultConfig();
9057 VideoStream config2 = config1;
9058 config2.scale_resolution_down_by *= 2;
9059
9060 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9061}
9062
9063TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9064 VideoStream config1 = DefaultConfig();
9065 VideoStream config2 = config1;
9066 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9067
9068 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9069}
9070
9071TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9072 VideoStream config1 = DefaultConfig();
9073 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009074 config2.scalability_mode = ScalabilityMode::kL2T1;
9075
9076 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9077}
9078
9079TEST_F(ReconfigureEncoderTest,
9080 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9081 VideoStream config1 = DefaultConfig();
9082 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009083 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009084 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009085
9086 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9087}
9088
Tommi62b01db2022-01-25 23:41:22 +01009089// Simple test that just creates and then immediately destroys an encoder.
9090// The purpose of the test is to make sure that nothing bad happens if the
9091// initialization step on the encoder queue, doesn't run.
9092TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9093 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9094 public:
9095 SuperLazyTaskQueue() = default;
9096 ~SuperLazyTaskQueue() override = default;
9097
9098 private:
9099 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009100 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009101 // meh.
9102 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009103 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9104 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009105 ASSERT_TRUE(false);
9106 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009107 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9108 TimeDelta delay) override {
9109 ADD_FAILURE();
9110 }
Tommi62b01db2022-01-25 23:41:22 +01009111 };
9112
9113 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009114 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009115 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009116 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9117 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009118 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009119 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9120 time_controller.GetClock());
9121 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9122 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9123 CreateBuiltinVideoBitrateAllocatorFactory();
9124 VideoStreamEncoderSettings encoder_settings{
9125 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9126 encoder_settings.encoder_factory = &encoder_factory;
9127 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9128
9129 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9130 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9131
9132 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9133 encoder_queue(new SuperLazyTaskQueue());
9134
9135 // Construct a VideoStreamEncoder instance and let it go out of scope without
9136 // doing anything else (including calling Stop()). This should be fine since
9137 // the posted init task will simply be deleted.
9138 auto encoder = std::make_unique<VideoStreamEncoder>(
9139 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009140 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9141 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009142 std::move(adapter), std::move(encoder_queue),
9143 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009144 kVideoBitrateAllocation,
9145 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009146
9147 // Stop the encoder explicitly. This additional step tests if we could
9148 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9149 // any more tasks.
9150 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009151}
9152
Markus Handellb4e96d42021-11-05 12:00:55 +01009153TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9154 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9155 auto* adapter_ptr = adapter.get();
9156 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009157 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9158 nullptr;
9159 EXPECT_CALL(*adapter_ptr, Initialize)
9160 .WillOnce(Invoke([&video_stream_encoder_callback](
9161 FrameCadenceAdapterInterface::Callback* callback) {
9162 video_stream_encoder_callback = callback;
9163 }));
9164 TaskQueueBase* encoder_queue = nullptr;
9165 auto video_stream_encoder =
9166 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009167
Markus Handelle59fee82021-12-23 09:29:23 +01009168 // First a call before we know the frame size and hence cannot compute the
9169 // number of simulcast layers.
9170 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9171 &FrameCadenceAdapterInterface::
9172 ZeroHertzModeParams::num_simulcast_layers,
9173 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009174 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009175 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009176 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9177 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009178 factory.DepleteTaskQueues();
9179
9180 // Then a call as we've computed the number of simulcast layers after a passed
9181 // frame.
9182 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9183 &FrameCadenceAdapterInterface::
9184 ZeroHertzModeParams::num_simulcast_layers,
9185 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009186 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009187 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009188 Mock::VerifyAndClearExpectations(adapter_ptr);
9189
Markus Handelle59fee82021-12-23 09:29:23 +01009190 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009191 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009192 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009193 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009194 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9195 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009196 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009197 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009198}
9199
9200TEST(VideoStreamEncoderFrameCadenceTest,
9201 ForwardsFramesIntoFrameCadenceAdapter) {
9202 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9203 auto* adapter_ptr = adapter.get();
9204 test::FrameForwarder video_source;
9205 SimpleVideoStreamEncoderFactory factory;
9206 auto video_stream_encoder = factory.Create(std::move(adapter));
9207 video_stream_encoder->SetSource(
9208 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9209
9210 EXPECT_CALL(*adapter_ptr, OnFrame);
9211 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9212 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009213 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009214}
9215
Markus Handellee225432021-11-29 12:35:12 +01009216TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9217 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9218 auto* adapter_ptr = adapter.get();
9219 test::FrameForwarder video_source;
9220 SimpleVideoStreamEncoderFactory factory;
9221 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9222 nullptr;
9223 EXPECT_CALL(*adapter_ptr, Initialize)
9224 .WillOnce(Invoke([&video_stream_encoder_callback](
9225 FrameCadenceAdapterInterface::Callback* callback) {
9226 video_stream_encoder_callback = callback;
9227 }));
9228 TaskQueueBase* encoder_queue = nullptr;
9229 auto video_stream_encoder =
9230 factory.Create(std::move(adapter), &encoder_queue);
9231
9232 // This is just to make the VSE operational. We'll feed a frame directly by
9233 // the callback interface.
9234 video_stream_encoder->SetSource(
9235 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9236
9237 VideoEncoderConfig video_encoder_config;
9238 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9239 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9240 /*max_data_payload_length=*/1000);
9241
9242 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9243 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009244 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009245 factory.DepleteTaskQueues();
9246}
9247
Markus Handell8d87c462021-12-16 11:37:16 +01009248TEST(VideoStreamEncoderFrameCadenceTest,
9249 DeactivatesActivatesLayersOnBitrateChanges) {
9250 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9251 auto* adapter_ptr = adapter.get();
9252 SimpleVideoStreamEncoderFactory factory;
9253 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9254 nullptr;
9255 EXPECT_CALL(*adapter_ptr, Initialize)
9256 .WillOnce(Invoke([&video_stream_encoder_callback](
9257 FrameCadenceAdapterInterface::Callback* callback) {
9258 video_stream_encoder_callback = callback;
9259 }));
9260 TaskQueueBase* encoder_queue = nullptr;
9261 auto video_stream_encoder =
9262 factory.Create(std::move(adapter), &encoder_queue);
9263
9264 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9265 // {150000, 450000}.
9266 VideoEncoderConfig video_encoder_config;
9267 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9268 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9269 kMaxPayloadLength);
9270 // Ensure an encoder is created.
9271 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9272
9273 // Both layers enabled at 1 MBit/s.
9274 video_stream_encoder->OnBitrateUpdated(
9275 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9276 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9277 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9278 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9279 factory.DepleteTaskQueues();
9280 Mock::VerifyAndClearExpectations(adapter_ptr);
9281
9282 // Layer 1 disabled at 200 KBit/s.
9283 video_stream_encoder->OnBitrateUpdated(
9284 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9285 DataRate::KilobitsPerSec(200), 0, 0, 0);
9286 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9287 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9288 factory.DepleteTaskQueues();
9289 Mock::VerifyAndClearExpectations(adapter_ptr);
9290
9291 // All layers off at suspended video.
9292 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9293 DataRate::Zero(), 0, 0, 0);
9294 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9295 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9296 factory.DepleteTaskQueues();
9297 Mock::VerifyAndClearExpectations(adapter_ptr);
9298
9299 // Both layers enabled again back at 1 MBit/s.
9300 video_stream_encoder->OnBitrateUpdated(
9301 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9302 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9303 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9304 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9305 factory.DepleteTaskQueues();
9306}
9307
9308TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9309 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9310 auto* adapter_ptr = adapter.get();
9311 SimpleVideoStreamEncoderFactory factory;
9312 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9313 nullptr;
9314 EXPECT_CALL(*adapter_ptr, Initialize)
9315 .WillOnce(Invoke([&video_stream_encoder_callback](
9316 FrameCadenceAdapterInterface::Callback* callback) {
9317 video_stream_encoder_callback = callback;
9318 }));
9319 TaskQueueBase* encoder_queue = nullptr;
9320 auto video_stream_encoder =
9321 factory.Create(std::move(adapter), &encoder_queue);
9322
9323 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9324 VideoEncoderConfig video_encoder_config;
9325 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9326 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9327 kMaxPayloadLength);
9328 video_stream_encoder->OnBitrateUpdated(
9329 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9330 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9331
9332 // Pass a frame which has unconverged results.
9333 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9334 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9335 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9336 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9337 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9338 CodecSpecificInfo codec_specific;
9339 codec_specific.codecType = kVideoCodecGeneric;
9340 return codec_specific;
9341 }));
9342 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9343 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9344 factory.DepleteTaskQueues();
9345 Mock::VerifyAndClearExpectations(adapter_ptr);
9346 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9347
9348 // Pass a frame which converges in layer 0 and not in layer 1.
9349 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9350 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9351 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9352 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9353 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9354 CodecSpecificInfo codec_specific;
9355 codec_specific.codecType = kVideoCodecGeneric;
9356 return codec_specific;
9357 }));
9358 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9359 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9360 factory.DepleteTaskQueues();
9361 Mock::VerifyAndClearExpectations(adapter_ptr);
9362 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9363}
9364
Markus Handell2e0f4f02021-12-21 19:14:58 +01009365TEST(VideoStreamEncoderFrameCadenceTest,
9366 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9367 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9368 auto* adapter_ptr = adapter.get();
9369 MockVideoSourceInterface mock_source;
9370 SimpleVideoStreamEncoderFactory factory;
9371 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9372 nullptr;
9373 EXPECT_CALL(*adapter_ptr, Initialize)
9374 .WillOnce(Invoke([&video_stream_encoder_callback](
9375 FrameCadenceAdapterInterface::Callback* callback) {
9376 video_stream_encoder_callback = callback;
9377 }));
9378 TaskQueueBase* encoder_queue = nullptr;
9379 auto video_stream_encoder =
9380 factory.Create(std::move(adapter), &encoder_queue);
9381 video_stream_encoder->SetSource(
9382 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9383 VideoEncoderConfig config;
9384 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9385 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9386 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9387 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9388 // Ensure the encoder is set up.
9389 factory.DepleteTaskQueues();
9390
Markus Handell818e7fb2021-12-30 13:01:33 +01009391 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9392 .WillOnce(Invoke([video_stream_encoder_callback] {
9393 video_stream_encoder_callback->RequestRefreshFrame();
9394 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009395 EXPECT_CALL(mock_source, RequestRefreshFrame);
9396 video_stream_encoder->SendKeyFrame();
9397 factory.DepleteTaskQueues();
9398 Mock::VerifyAndClearExpectations(adapter_ptr);
9399 Mock::VerifyAndClearExpectations(&mock_source);
9400
Markus Handell818e7fb2021-12-30 13:01:33 +01009401 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009402 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9403 video_stream_encoder->SendKeyFrame();
9404 factory.DepleteTaskQueues();
9405}
9406
Markus Handell818e7fb2021-12-30 13:01:33 +01009407TEST(VideoStreamEncoderFrameCadenceTest,
9408 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9409 SimpleVideoStreamEncoderFactory factory;
9410 auto encoder_queue =
9411 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9412 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9413
9414 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009415 test::ScopedKeyValueConfig field_trials(
9416 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009417 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009418 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9419 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009420 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9421
9422 MockVideoSourceInterface mock_source;
9423 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009424 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009425
9426 video_stream_encoder->SetSource(
9427 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9428 VideoEncoderConfig config;
9429 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9430 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9431 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9432
9433 // Eventually expect a refresh frame request when requesting a key frame
9434 // before initializing zero-hertz mode. This can happen in reality because the
9435 // threads invoking key frame requests and constraints setup aren't
9436 // synchronized.
9437 EXPECT_CALL(mock_source, RequestRefreshFrame);
9438 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009439 constexpr int kMaxFps = 30;
9440 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9441 factory.GetTimeController()->AdvanceTime(
9442 TimeDelta::Seconds(1) *
9443 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9444 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009445}
9446
perkj26091b12016-09-01 01:17:40 -07009447} // namespace webrtc